[Hardware] Support platforms and plugin system (#774)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
b11d436e19
commit
c9392dff01
103 files changed
+1770
-1218
No files matched your search
@@ -2,14 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker classes for diffusion models."""
|
||||
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_worker import (
|
||||
GPUDiffusionWorker,
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.diffusion_worker import (
|
||||
DiffusionWorker,
|
||||
WorkerProc,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"GPUDiffusionModelRunner",
|
||||
"GPUDiffusionWorker",
|
||||
"DiffusionModelRunner",
|
||||
"DiffusionWorker",
|
||||
"WorkerProc",
|
||||
]
|
||||
+1
-1
@@ -34,7 +34,7 @@ from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GPUDiffusionModelRunner:
|
||||
class DiffusionModelRunner:
|
||||
"""
|
||||
Model runner that handles model loading and execution for diffusion models.
|
||||
|
||||
+18
-12
@@ -4,7 +4,7 @@
|
||||
Diffusion Worker for vLLM-Omni.
|
||||
|
||||
Handles GPU infrastructure initialization and delegates model operations
|
||||
to GPUDiffusionModelRunner.
|
||||
to DiffusionModelRunner.
|
||||
"""
|
||||
|
||||
import multiprocessing as mp
|
||||
@@ -31,13 +31,14 @@ from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
|
||||
from vllm_omni.diffusion.profiler import CurrentProfiler
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GPUDiffusionWorker:
|
||||
class DiffusionWorker:
|
||||
"""
|
||||
A worker that manages GPU infrastructure and delegates to the model runner.
|
||||
|
||||
@@ -47,7 +48,7 @@ class GPUDiffusionWorker:
|
||||
- Memory management (sleep/wake)
|
||||
|
||||
All model-related operations (loading, compilation, execution) are
|
||||
delegated to GPUDiffusionModelRunner.
|
||||
delegated to DiffusionModelRunner.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -61,7 +62,7 @@ class GPUDiffusionWorker:
|
||||
self.od_config = od_config
|
||||
self.device: torch.device | None = None
|
||||
self.vllm_config: VllmConfig | None = None
|
||||
self.model_runner: GPUDiffusionModelRunner | None = None
|
||||
self.model_runner: DiffusionModelRunner | None = None
|
||||
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
|
||||
self.lora_manager: DiffusionLoRAManager | None = None
|
||||
self.init_device()
|
||||
@@ -79,8 +80,8 @@ class GPUDiffusionWorker:
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
|
||||
# Setup device
|
||||
self.device = torch.device(f"cuda:{rank}")
|
||||
torch.cuda.set_device(self.device)
|
||||
self.device = current_omni_platform.get_torch_device(rank)
|
||||
current_omni_platform.set_device(self.device)
|
||||
|
||||
# Create vllm_config for parallel configuration
|
||||
vllm_config = VllmConfig()
|
||||
@@ -105,7 +106,7 @@ class GPUDiffusionWorker:
|
||||
)
|
||||
|
||||
# Create model runner and load model
|
||||
self.model_runner = GPUDiffusionModelRunner(
|
||||
self.model_runner = DiffusionModelRunner(
|
||||
vllm_config=self.vllm_config,
|
||||
od_config=self.od_config,
|
||||
device=self.device,
|
||||
@@ -176,7 +177,7 @@ class GPUDiffusionWorker:
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
|
||||
free_bytes_before_sleep = current_omni_platform.get_free_memory()
|
||||
|
||||
# Save the buffers before level 2 sleep
|
||||
if level == 2 and self.model_runner is not None:
|
||||
@@ -185,7 +186,9 @@ class GPUDiffusionWorker:
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
free_bytes_after_sleep = current_omni_platform.get_free_memory()
|
||||
device_id = self.device.index if self.device.index is not None else 0
|
||||
total = current_omni_platform.get_device_total_memory(device_id)
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
used_bytes = total - free_bytes_after_sleep
|
||||
assert freed_bytes >= 0, "Memory usage increased after sleeping."
|
||||
@@ -270,9 +273,9 @@ class WorkerProc:
|
||||
self.gpu_id = gpu_id
|
||||
self._running = True
|
||||
|
||||
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> GPUDiffusionWorker:
|
||||
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> DiffusionWorker:
|
||||
"""Create a worker instance. Override in subclasses for different worker types."""
|
||||
return GPUDiffusionWorker(
|
||||
return DiffusionWorker(
|
||||
local_rank=gpu_id,
|
||||
rank=gpu_id,
|
||||
od_config=od_config,
|
||||
@@ -379,6 +382,9 @@ class WorkerProc:
|
||||
broadcast_handle,
|
||||
) -> None:
|
||||
"""Worker initialization and execution loops."""
|
||||
from vllm_omni.plugins import load_omni_general_plugins
|
||||
|
||||
load_omni_general_plugins()
|
||||
worker_proc = WorkerProc(
|
||||
od_config,
|
||||
gpu_id=rank,
|
||||
@@ -1,7 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NPU worker classes for diffusion models."""
|
||||
|
||||
from vllm_omni.diffusion.worker.npu.npu_worker import NPUWorker, NPUWorkerProc
|
||||
|
||||
__all__ = ["NPUWorker", "NPUWorkerProc"]
|
||||
@@ -1,255 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
|
||||
import torch
|
||||
from vllm.config import LoadConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes
|
||||
|
||||
from vllm_omni.diffusion.cache.selector import get_cache_backend
|
||||
from vllm_omni.diffusion.data import (
|
||||
DiffusionOutput,
|
||||
OmniDiffusionConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
destroy_distributed_env,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_worker import WorkerProc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NPUWorker:
|
||||
"""
|
||||
A worker that executes the model on a single NPU.
|
||||
Inherits from GPUWorker and overrides device-specific initialization.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
local_rank: int,
|
||||
rank: int,
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
self.local_rank = local_rank
|
||||
self.rank = rank
|
||||
self.od_config = od_config
|
||||
self.pipeline = None
|
||||
self.device = None
|
||||
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
|
||||
self.init_device_and_model()
|
||||
|
||||
def init_device_and_model(self) -> None:
|
||||
"""Initialize the NPU device and load the model."""
|
||||
world_size = self.od_config.num_gpus
|
||||
rank = self.rank
|
||||
# Set environment variables for distributed initialization
|
||||
os.environ["MASTER_ADDR"] = "localhost"
|
||||
os.environ["MASTER_PORT"] = str(self.od_config.master_port)
|
||||
os.environ["LOCAL_RANK"] = str(self.local_rank)
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
|
||||
device = torch.device(f"npu:{rank}")
|
||||
torch.npu.set_device(device)
|
||||
|
||||
# hack
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = self.od_config.parallel_config.tensor_parallel_size
|
||||
vllm_config.parallel_config.data_parallel_size = self.od_config.parallel_config.data_parallel_size
|
||||
self.vllm_config = vllm_config
|
||||
with set_forward_context(vllm_config=vllm_config, omni_diffusion_config=self.od_config):
|
||||
init_distributed_environment(world_size=world_size, rank=rank)
|
||||
logger.info(f"Worker {self.rank}: Initialized device and distributed environment.")
|
||||
parallel_config = self.od_config.parallel_config
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=parallel_config.data_parallel_size,
|
||||
cfg_parallel_size=parallel_config.cfg_parallel_size,
|
||||
sequence_parallel_size=parallel_config.sequence_parallel_size,
|
||||
ulysses_degree=parallel_config.ulysses_degree,
|
||||
ring_degree=parallel_config.ring_degree,
|
||||
tensor_parallel_size=parallel_config.tensor_parallel_size,
|
||||
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
|
||||
)
|
||||
|
||||
load_config = LoadConfig()
|
||||
model_loader = DiffusersPipelineLoader(load_config)
|
||||
time_before_load = time.perf_counter()
|
||||
with DeviceMemoryProfiler() as m:
|
||||
self.pipeline = model_loader.load_model(
|
||||
od_config=self.od_config,
|
||||
load_device=f"npu:{rank}",
|
||||
)
|
||||
time_after_load = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Model loading took %.4f GiB and %.6f seconds",
|
||||
m.consumed_memory / GiB_bytes,
|
||||
time_after_load - time_before_load,
|
||||
)
|
||||
logger.info(f"Worker {self.rank}: Model loaded successfully.")
|
||||
|
||||
# Setup cache backend based on type (both backends use enable()/reset() interface)
|
||||
self.cache_backend = get_cache_backend(self.od_config.cache_backend, self.od_config.cache_config)
|
||||
|
||||
if self.cache_backend is not None:
|
||||
self.cache_backend.enable(self.pipeline)
|
||||
|
||||
def generate(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput:
|
||||
"""
|
||||
Generate output for the given requests.
|
||||
|
||||
Args:
|
||||
requests: List of diffusion requests
|
||||
|
||||
Returns:
|
||||
DiffusionOutput with generated results
|
||||
"""
|
||||
return self.execute_model(requests, self.od_config)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput:
|
||||
"""
|
||||
Execute a forward pass.
|
||||
"""
|
||||
assert self.pipeline is not None
|
||||
if not reqs or len(reqs) == 0:
|
||||
raise ValueError("Cannot execute model with empty request list")
|
||||
# TODO: dealing with first req for now
|
||||
req = reqs[0]
|
||||
|
||||
if req.generator is None and req.seed is not None:
|
||||
req.generator = torch.Generator(device=self.device).manual_seed(req.seed)
|
||||
|
||||
# Refresh cache context if needed
|
||||
if (
|
||||
not getattr(req, "skip_cache_refresh", False)
|
||||
and self.cache_backend is not None
|
||||
and self.cache_backend.is_enabled()
|
||||
):
|
||||
self.cache_backend.refresh(self.pipeline, req.num_inference_steps)
|
||||
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config):
|
||||
output = self.pipeline.forward(req)
|
||||
return output
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return self.pipeline.load_weights(weights)
|
||||
|
||||
def sleep(self, level: int = 1) -> bool:
|
||||
"""
|
||||
Put the worker to sleep. The worker should not process any requests.
|
||||
The caller should guarantee that no requests are being processed
|
||||
during the sleep period, before `wake_up` is called.
|
||||
|
||||
Args:
|
||||
level: The sleep level. Level 1 sleep will offload the model
|
||||
weights and discard the kv cache.
|
||||
Currently only support level 1.
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
|
||||
|
||||
# Save the buffers before level 2 sleep
|
||||
if level == 2:
|
||||
model = self.pipeline
|
||||
self._sleep_saved_buffers = {name: buffer.cpu().clone() for name, buffer in model.named_buffers()}
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
used_bytes = total - free_bytes_after_sleep
|
||||
assert freed_bytes >= 0, "Memory usage increased after sleeping."
|
||||
logger.info(
|
||||
"Sleep mode freed %.2f GiB memory, %.2f GiB memory is still in use.",
|
||||
freed_bytes / GiB_bytes,
|
||||
used_bytes / GiB_bytes,
|
||||
)
|
||||
return True
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
"""
|
||||
Wake up the worker from sleep mode. See the sleep function
|
||||
method for more details.
|
||||
|
||||
Args:
|
||||
tags: An optional list of tags to reallocate the worker memory
|
||||
for specific memory allocations. Values must be in
|
||||
`("weights")`. If None, all memory is reallocated.
|
||||
wake_up should be called with all tags (or None) before the
|
||||
worker is used again.
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator.wake_up(tags)
|
||||
|
||||
# Restore the buffers after level 2 sleep
|
||||
if len(self._sleep_saved_buffers):
|
||||
model = self.pipeline
|
||||
for name, buffer in model.named_buffers():
|
||||
if name in self._sleep_saved_buffers:
|
||||
buffer.data.copy_(self._sleep_saved_buffers[name].data)
|
||||
self._sleep_saved_buffers = {}
|
||||
return True
|
||||
|
||||
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
|
||||
if self.od_config.enable_sleep_mode:
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
if tag == "weights":
|
||||
assert allocator.get_current_usage() == 0, "Sleep mode can only be used for one instance per process."
|
||||
return allocator.use_memory_pool(tag=tag)
|
||||
else:
|
||||
return nullcontext()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
destroy_distributed_env()
|
||||
|
||||
|
||||
class NPUWorkerProc(WorkerProc):
|
||||
"""Wrapper that runs one NPUWorker in a separate process."""
|
||||
|
||||
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> NPUWorker:
|
||||
"""Create an NPUWorker instead of GPUWorker."""
|
||||
return NPUWorker(
|
||||
local_rank=gpu_id,
|
||||
rank=gpu_id,
|
||||
od_config=od_config,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def worker_main(
|
||||
rank: int,
|
||||
od_config: OmniDiffusionConfig,
|
||||
pipe_writer: mp.connection.Connection,
|
||||
broadcast_handle,
|
||||
) -> None:
|
||||
"""Worker initialization and execution loops."""
|
||||
|
||||
worker_proc = NPUWorkerProc(
|
||||
od_config,
|
||||
gpu_id=rank,
|
||||
broadcast_handle=broadcast_handle,
|
||||
)
|
||||
logger.info(f"Worker {rank}: Scheduler loop started.")
|
||||
pipe_writer.send(
|
||||
{
|
||||
"status": "ready",
|
||||
"result_handle": worker_proc.result_mq_handle if rank == 0 else None,
|
||||
}
|
||||
)
|
||||
worker_proc.worker_busy_loop()
|
||||
logger.info(f"Worker {rank}: Shutdown complete.")
|
||||
Reference in new issue
Block a user