Refactor GPU diffusion model runner and worker
- Introduced `DiffusionModelRunner` class to handle diffusion processes, extending `OmniGPUModelRunner`. - Updated `DiffusionGPUWorker` to initialize device settings and manage memory more effectively. - Enhanced model execution with support for multimodal inputs and improved tensor management. - Removed legacy code and streamlined the generation process for better performance.
This commit is contained in:
1 parent
7a85034d9b
commit
811057ce11
2 files changed
+414
-160
No files matched your search
@@ -1,56 +1,67 @@
|
||||
"""Worker abstraction for executing diffusion model runners."""
|
||||
from vllm.v1.worker.gpu_worker import Worker as GPUWorker
|
||||
import torch
|
||||
import os
|
||||
import gc
|
||||
from vllm.utils import GiB_bytes, MemorySnapshot
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.worker.worker import _check_if_gpu_supports_dtype
|
||||
from vllm.v1.worker.gpu_worker import init_worker_distributed_environment
|
||||
from vllm.model_executor import set_random_seed
|
||||
from vllm.v1.utils import report_usage_stats
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Optional
|
||||
|
||||
from vllm.v1.worker.worker_base import WorkerBase
|
||||
|
||||
from .gpu_diffusion_model_runner import DiffusionModelRunner, DiffusionRunnerOutput
|
||||
from vllm_omni.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
|
||||
|
||||
class DiffusionGPUWorker(WorkerBase):
|
||||
"""Minimal worker wrapping DiffusionModelRunner.
|
||||
class DiffusionGPUWorker(GPUWorker):
|
||||
def init_device(self):
|
||||
if self.device_config.device.type == "cuda":
|
||||
# torch.distributed.all_reduce does not free the input tensor until
|
||||
# the synchronization point. This causes the memory usage to grow
|
||||
# as the number of all_reduce calls increases. This env var disables
|
||||
# this behavior.
|
||||
# Related issue:
|
||||
# https://discuss.pytorch.org/t/cuda-allocation-lifetime-for-inputs-to-distributed-all-reduce/191573
|
||||
os.environ["TORCH_NCCL_AVOID_RECORD_STREAMS"] = "1"
|
||||
|
||||
The worker interface mirrors the shape expected by vLLM executors: it owns
|
||||
a model runner instance and exposes a `generate` method that can be called
|
||||
by an executor implementation.
|
||||
"""
|
||||
# This env var set by Ray causes exceptions with graph building.
|
||||
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
|
||||
self.device = torch.device(f"cuda:{self.local_rank}")
|
||||
current_platform.set_device(self.device)
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
model_path: str,
|
||||
*,
|
||||
pipeline_name: Optional[str] = None,
|
||||
device: Optional[str] = None,
|
||||
dtype: Optional[str] = None,
|
||||
) -> None:
|
||||
self.model_runner = DiffusionModelRunner(
|
||||
model_path,
|
||||
pipeline_name=pipeline_name,
|
||||
device=device,
|
||||
dtype=dtype,
|
||||
)
|
||||
_check_if_gpu_supports_dtype(self.model_config.dtype)
|
||||
gc.collect()
|
||||
torch.cuda.empty_cache()
|
||||
|
||||
def generate(
|
||||
self,
|
||||
prompt: str,
|
||||
*,
|
||||
height: int = 512,
|
||||
width: int = 512,
|
||||
num_inference_steps: int = 30,
|
||||
guidance_scale: float = 5.0,
|
||||
negative_prompt: Optional[str] = None,
|
||||
seed: Optional[int] = None,
|
||||
image: Optional[Any] = None,
|
||||
) -> DiffusionRunnerOutput:
|
||||
return self.model_runner.generate(
|
||||
prompt=prompt,
|
||||
height=height,
|
||||
width=width,
|
||||
num_inference_steps=num_inference_steps,
|
||||
guidance_scale=guidance_scale,
|
||||
negative_prompt=negative_prompt,
|
||||
seed=seed,
|
||||
image=image,
|
||||
)
|
||||
# take current memory snapshot
|
||||
self.init_snapshot = MemorySnapshot()
|
||||
self.requested_memory = (self.init_snapshot.total_memory *
|
||||
self.cache_config.gpu_memory_utilization)
|
||||
if self.init_snapshot.free_memory < self.requested_memory:
|
||||
GiB = lambda b: round(b / GiB_bytes, 2)
|
||||
raise ValueError(
|
||||
f"Free memory on device "
|
||||
f"({GiB(self.init_snapshot.free_memory)}/"
|
||||
f"{GiB(self.init_snapshot.total_memory)} GiB) on startup "
|
||||
f"is less than desired GPU memory utilization "
|
||||
f"({self.cache_config.gpu_memory_utilization}, "
|
||||
f"{GiB(self.requested_memory)} GiB). Decrease GPU memory "
|
||||
f"utilization or reduce GPU memory used by other processes."
|
||||
)
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Not support device type: {self.device_config.device}")
|
||||
# Initialize the distributed environment.
|
||||
init_worker_distributed_environment(self.vllm_config, self.rank,
|
||||
self.distributed_init_method,
|
||||
self.local_rank,
|
||||
current_platform.dist_backend)
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Construct the model runner
|
||||
self.model_runner: DiffusionModelRunner = DiffusionModelRunner(
|
||||
self.vllm_config, self.device)
|
||||
|
||||
if self.rank == 0:
|
||||
# If usage stat is enabled, collect relevant info.
|
||||
report_usage_stats(self.vllm_config)
|
||||
Reference in new issue
Block a user