[Diffusion]: Diffusion Ulysses-Sequence-Parallelism support (#189)

Signed-off-by: Didan Deng <33117903+wtomin@users.noreply.github.com>
This commit is contained in:
Didan Deng authored and GitHub committed 2025-12-17 15:41:31 +08:00
1 parent f785a5d401
commit 85f0c1177f
21 files changed
+3513 -61

No files matched your search

+32 -24
View File
@@ -8,10 +8,6 @@ import torch
import zmq
from vllm.config import LoadConfig, VllmConfig, set_current_vllm_config
from vllm.distributed.device_communicators.shm_broadcast import MessageQueue
from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.logger import init_logger
from vllm.utils import DeviceMemoryProfiler, GiB_bytes
@@ -20,6 +16,12 @@ from vllm_omni.diffusion.data import (
SHUTDOWN_MESSAGE,
DiffusionOutput,
OmniDiffusionConfig,
set_current_omni_diffusion_config,
)
from vllm_omni.diffusion.distributed.parallel_state import (
destroy_distributed_env,
init_distributed_environment,
initialize_model_parallel,
)
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.request import OmniDiffusionRequest
@@ -61,22 +63,33 @@ class GPUWorker:
# hack
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = self.od_config.num_gpus
set_current_vllm_config(vllm_config)
vllm_config.parallel_config.tensor_parallel_size = self.od_config.parallel_config.tensor_parallel_size
vllm_config.parallel_config.data_parallel_size = self.od_config.parallel_config.data_parallel_size
init_distributed_environment(world_size=world_size, rank=rank)
initialize_model_parallel(tensor_model_parallel_size=world_size)
logger.info(f"Worker {self.rank}: Initialized device and distributed environment.")
with set_current_omni_diffusion_config(self.od_config):
with set_current_vllm_config(vllm_config):
init_distributed_environment(world_size=world_size, rank=rank)
logger.info(f"Worker {self.rank}: Initialized device and distributed environment.")
parallel_config = self.od_config.parallel_config
initialize_model_parallel(
data_parallel_size=parallel_config.data_parallel_size,
cfg_parallel_size=parallel_config.cfg_parallel_size,
sequence_parallel_size=parallel_config.sequence_parallel_size,
ulysses_degree=parallel_config.ulysses_degree,
ring_degree=parallel_config.ring_degree,
tensor_parallel_size=parallel_config.tensor_parallel_size,
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
)
load_config = LoadConfig()
model_loader = DiffusersPipelineLoader(load_config)
time_before_load = time.perf_counter()
with DeviceMemoryProfiler() as m:
self.pipeline = model_loader.load_model(
od_config=self.od_config,
load_device=f"cuda:{rank}",
)
time_after_load = time.perf_counter()
load_config = LoadConfig()
model_loader = DiffusersPipelineLoader(load_config)
time_before_load = time.perf_counter()
with DeviceMemoryProfiler() as m:
self.pipeline = model_loader.load_model(
od_config=self.od_config,
load_device=f"cuda:{rank}",
)
time_after_load = time.perf_counter()
logger.info(
"Model loading took %.4f GiB and %.6f seconds",
@@ -108,12 +121,7 @@ class GPUWorker:
return output
def shutdown(self) -> None:
if torch.distributed.is_initialized():
try:
torch.distributed.destroy_process_group()
logger.info("Worker %s: Destroyed process group", self.rank)
except Exception as exc: # pragma: no cover - best effort cleanup
logger.warning("Worker %s: Failed to destroy process group: %s", self.rank, exc)
destroy_distributed_env()
class WorkerProc: