[Hardware] Support platforms and plugin system (#774)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
b11d436e19
commit
c9392dff01
103 files changed
+1770
-1218
No files matched your search
@@ -7,12 +7,18 @@ from typing import Generic, TypeVar
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
class AttentionBackend(ABC):
|
||||
"""Abstract class for diffusion attention backends."""
|
||||
|
||||
accept_output_buffer: bool = False
|
||||
|
||||
@classmethod
|
||||
def supports_attention_mask(cls) -> bool:
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
@abstractmethod
|
||||
def get_name() -> str:
|
||||
@@ -77,12 +83,58 @@ class AttentionImpl(ABC, Generic[T]):
|
||||
) -> None:
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T,
|
||||
attn_metadata: T | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Dispatch to platform-specific forward implementation."""
|
||||
if current_omni_platform.is_rocm():
|
||||
return self.forward_hip(query, key, value, attn_metadata)
|
||||
elif current_omni_platform.is_cuda():
|
||||
return self.forward_cuda(query, key, value, attn_metadata)
|
||||
elif current_omni_platform.is_npu():
|
||||
return self.forward_npu(query, key, value, attn_metadata)
|
||||
elif current_omni_platform.is_xpu():
|
||||
return self.forward_xpu(query, key, value, attn_metadata)
|
||||
else:
|
||||
raise NotImplementedError(f"No forward implementation for platform: {current_omni_platform}")
|
||||
|
||||
def forward_cuda(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_npu(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T | None = None,
|
||||
) -> torch.Tensor:
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_hip(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T | None = None,
|
||||
) -> torch.Tensor:
|
||||
# By default, HIP ops are compatible with CUDA ops.
|
||||
return self.forward_cuda(query, key, value, attn_metadata)
|
||||
@@ -1,92 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from importlib.util import find_spec
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
AttentionImpl,
|
||||
AttentionMetadata,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class AscendAttentionBackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
return [x for x in range(1024)] # todo
|
||||
|
||||
@staticmethod
|
||||
def get_name() -> str:
|
||||
return "ASCEND"
|
||||
|
||||
@staticmethod
|
||||
def get_impl_cls() -> type["AscendAttentionBackendImpl"]:
|
||||
return AscendAttentionBackendImpl
|
||||
|
||||
|
||||
class AscendAttentionBackendImpl(AttentionImpl):
|
||||
def __init__(
|
||||
self,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
softmax_scale: float,
|
||||
causal: bool = False,
|
||||
num_kv_heads: int | None = None,
|
||||
prefix: str = "",
|
||||
**extra_impl_args,
|
||||
) -> None:
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||
|
||||
output = torch.nn.functional.scaled_dot_product_attention(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
dropout_p=0.0,
|
||||
is_causal=self.causal,
|
||||
scale=self.softmax_scale,
|
||||
)
|
||||
out = output.permute(0, 2, 1, 3)
|
||||
return out
|
||||
|
||||
def forward(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
if find_spec("mindiesd"):
|
||||
from mindiesd import attention_forward
|
||||
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||
output = attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
opt_mode="manual",
|
||||
op_type="fused_attn_score",
|
||||
layout="BNSD",
|
||||
)
|
||||
return output
|
||||
else:
|
||||
return self.forward_native(query, key, value, attn_metadata)
|
||||
@@ -9,13 +9,6 @@ from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionImpl,
|
||||
AttentionMetadata,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.utils.fa import (
|
||||
_pad_input,
|
||||
_unpad_input,
|
||||
_upad_input,
|
||||
flash_attn_func,
|
||||
flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -23,6 +16,10 @@ logger = init_logger(__name__)
|
||||
class FlashAttentionBackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
|
||||
@classmethod
|
||||
def supports_attention_mask(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
return [64, 96, 128, 192, 256]
|
||||
@@ -51,25 +48,22 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
def forward(
|
||||
def forward_cuda(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Flash attention implementation.
|
||||
"""CUDA/ROCm flash attention implementation."""
|
||||
from vllm_omni.diffusion.attention.backends.utils.fa import (
|
||||
_pad_input,
|
||||
_unpad_input,
|
||||
_upad_input,
|
||||
flash_attn_func,
|
||||
flash_attn_varlen_func,
|
||||
)
|
||||
|
||||
Args:
|
||||
query: (batch_size, seq_len, num_heads, head_dim)
|
||||
key: (batch_size, seq_len, num_heads, head_dim)
|
||||
value: (batch_size, seq_len, num_heads, head_dim)
|
||||
attn_metadata: AttentionMetadata. Attention mask is supported as attn_metadata.attn_mask
|
||||
|
||||
Returns:
|
||||
(batch_size, seq_len, num_heads, head_dim)
|
||||
"""
|
||||
query_length = query.size(1)
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata is not None else None
|
||||
# Contains at least one padding token in the sequence
|
||||
@@ -106,3 +100,25 @@ class FlashAttentionImpl(AttentionImpl):
|
||||
softmax_scale=self.softmax_scale,
|
||||
)
|
||||
return out
|
||||
|
||||
def forward_npu(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: AttentionMetadata = None,
|
||||
) -> torch.Tensor:
|
||||
"""NPU attention implementation using mindiesd."""
|
||||
from mindiesd import attention_forward
|
||||
|
||||
attention_mask = attn_metadata.attn_mask if attn_metadata else None
|
||||
output = attention_forward(
|
||||
query,
|
||||
key,
|
||||
value,
|
||||
attn_mask=attention_mask,
|
||||
opt_mode="manual",
|
||||
op_type="fused_attn_score",
|
||||
layout="BNSD",
|
||||
)
|
||||
return output
|
||||
@@ -0,0 +1,152 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Diffusion attention backend registry.
|
||||
|
||||
This module provides an enum-based registry for diffusion attention backends,
|
||||
similar to vLLM's AttentionBackendEnum. Each backend registers its class path,
|
||||
and platforms can override or extend backends using register_backend().
|
||||
"""
|
||||
|
||||
from collections.abc import Callable
|
||||
from enum import Enum, EnumMeta
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.import_utils import resolve_obj_by_qualname
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm_omni.diffusion.attention.backends.abstract import AttentionBackend
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class _DiffusionBackendEnumMeta(EnumMeta):
|
||||
"""Metaclass for DiffusionAttentionBackendEnum to provide better error messages."""
|
||||
|
||||
def __getitem__(cls, name: str) -> "DiffusionAttentionBackendEnum":
|
||||
"""Get backend by name with helpful error messages."""
|
||||
try:
|
||||
return super().__getitem__(name) # type: ignore[return-value]
|
||||
except KeyError:
|
||||
members = list(cls.__members__.keys())
|
||||
valid_backends = ", ".join(members)
|
||||
raise ValueError(
|
||||
f"Unknown diffusion attention backend: '{name}'. Valid options are: {valid_backends}"
|
||||
) from None
|
||||
|
||||
|
||||
class DiffusionAttentionBackendEnum(Enum, metaclass=_DiffusionBackendEnumMeta):
|
||||
"""Enumeration of all supported diffusion attention backends.
|
||||
|
||||
The enum value is the default class path, but this can be overridden
|
||||
at runtime using register_backend().
|
||||
|
||||
To get the actual backend class (respecting overrides), use:
|
||||
backend.get_class()
|
||||
|
||||
Example:
|
||||
# Get backend class
|
||||
backend = DiffusionAttentionBackendEnum.FLASH_ATTN
|
||||
backend_cls = backend.get_class()
|
||||
|
||||
# Register custom backend
|
||||
@register_diffusion_backend(DiffusionAttentionBackendEnum.CUSTOM)
|
||||
class MyCustomBackend:
|
||||
...
|
||||
"""
|
||||
|
||||
# Common backends (available on most platforms)
|
||||
FLASH_ATTN = "vllm_omni.diffusion.attention.backends.flash_attn.FlashAttentionBackend"
|
||||
TORCH_SDPA = "vllm_omni.diffusion.attention.backends.sdpa.SDPABackend"
|
||||
SAGE_ATTN = "vllm_omni.diffusion.attention.backends.sage_attn.SageAttentionBackend"
|
||||
|
||||
def get_path(self, include_classname: bool = True) -> str:
|
||||
"""Get the class path for this backend (respects overrides).
|
||||
|
||||
Returns:
|
||||
The fully qualified class path string
|
||||
|
||||
Raises:
|
||||
ValueError: If backend has empty path and is not registered
|
||||
"""
|
||||
path = _DIFFUSION_ATTN_OVERRIDES.get(self, self.value)
|
||||
if not path:
|
||||
raise ValueError(
|
||||
f"Backend {self.name} must be registered before use. "
|
||||
f"Use register_diffusion_backend(DiffusionAttentionBackendEnum.{self.name}, "
|
||||
f"'your.module.YourClass')"
|
||||
)
|
||||
if not include_classname:
|
||||
path = path.rsplit(".", 1)[0]
|
||||
return path
|
||||
|
||||
def get_class(self) -> "type[AttentionBackend]":
|
||||
"""Get the backend class (respects overrides).
|
||||
|
||||
Returns:
|
||||
The backend class
|
||||
|
||||
Raises:
|
||||
ImportError: If the backend class cannot be imported
|
||||
ValueError: If backend has empty path and is not registered
|
||||
"""
|
||||
return resolve_obj_by_qualname(self.get_path())
|
||||
|
||||
def is_overridden(self) -> bool:
|
||||
"""Check if this backend has been overridden.
|
||||
|
||||
Returns:
|
||||
True if the backend has a registered override
|
||||
"""
|
||||
return self in _DIFFUSION_ATTN_OVERRIDES
|
||||
|
||||
def clear_override(self) -> None:
|
||||
"""Clear any override for this backend, reverting to the default."""
|
||||
_DIFFUSION_ATTN_OVERRIDES.pop(self, None)
|
||||
|
||||
|
||||
# Override registry
|
||||
_DIFFUSION_ATTN_OVERRIDES: dict[DiffusionAttentionBackendEnum, str] = {}
|
||||
|
||||
|
||||
def register_diffusion_backend(
|
||||
backend: DiffusionAttentionBackendEnum,
|
||||
class_path: str | None = None,
|
||||
) -> Callable[[type], type]:
|
||||
"""Register or override a diffusion backend implementation.
|
||||
|
||||
Args:
|
||||
backend: The DiffusionAttentionBackendEnum member to register
|
||||
class_path: Optional class path. If not provided and used as
|
||||
decorator, will be auto-generated from the class.
|
||||
|
||||
Returns:
|
||||
Decorator function if class_path is None, otherwise a no-op
|
||||
|
||||
Examples:
|
||||
# Override an existing backend
|
||||
@register_diffusion_backend(DiffusionAttentionBackendEnum.FLASH_ATTN)
|
||||
class MyCustomFlashAttn:
|
||||
...
|
||||
|
||||
# Override an existing backend (e.g., ASCEND_ATTN)
|
||||
@register_diffusion_backend(DiffusionAttentionBackendEnum.ASCEND_ATTN)
|
||||
class CustomAscendAttentionBackend:
|
||||
...
|
||||
|
||||
# Direct registration
|
||||
register_diffusion_backend(
|
||||
DiffusionAttentionBackendEnum.CUSTOM,
|
||||
"my.module.MyCustomBackend"
|
||||
)
|
||||
"""
|
||||
|
||||
def decorator(cls: type) -> type:
|
||||
_DIFFUSION_ATTN_OVERRIDES[backend] = f"{cls.__module__}.{cls.__qualname__}"
|
||||
return cls
|
||||
|
||||
if class_path is not None:
|
||||
_DIFFUSION_ATTN_OVERRIDES[backend] = class_path
|
||||
return lambda x: x
|
||||
|
||||
return decorator
|
||||
@@ -54,7 +54,7 @@ class SageAttentionImpl(AttentionImpl):
|
||||
self.causal = causal
|
||||
self.softmax_scale = softmax_scale
|
||||
|
||||
def forward(
|
||||
def forward_cuda(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
|
||||
@@ -16,6 +16,10 @@ logger = init_logger(__name__)
|
||||
class SDPABackend(AttentionBackend):
|
||||
accept_output_buffer: bool = True
|
||||
|
||||
@classmethod
|
||||
def supports_attention_mask(cls) -> bool:
|
||||
return True
|
||||
|
||||
@staticmethod
|
||||
def get_supported_head_sizes() -> list[int]:
|
||||
return [x for x in range(1024)] # todo
|
||||
|
||||
@@ -15,9 +15,9 @@
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm_omni.utils.platform_utils import is_rocm
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if is_rocm():
|
||||
if current_omni_platform.is_rocm():
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
|
||||
# Choose to enable this by default on ROCm
|
||||
|
||||
@@ -1,52 +1,54 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Diffusion attention backend selector.
|
||||
|
||||
This module provides the interface for selecting diffusion attention backends.
|
||||
The actual backend selection logic is delegated to the platform layer
|
||||
(vllm_omni.platforms), similar to how vLLM handles attention backend selection.
|
||||
|
||||
Usage:
|
||||
from vllm_omni.diffusion.attention.selector import get_attn_backend
|
||||
|
||||
# Get the appropriate backend for current platform
|
||||
backend_cls = get_attn_backend(head_size=64)
|
||||
|
||||
# Or override via environment variable
|
||||
# export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
|
||||
"""
|
||||
|
||||
import importlib
|
||||
import os
|
||||
from functools import cache
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.abstract import (
|
||||
AttentionBackend,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_rocm
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# environment variable value -> backend module and class
|
||||
_BACKEND_CONFIG = {
|
||||
"FLASH_ATTN": {
|
||||
"module": "vllm_omni.diffusion.attention.backends.flash_attn",
|
||||
"class": "FlashAttentionBackend",
|
||||
},
|
||||
"TORCH_SDPA": {
|
||||
"module": "vllm_omni.diffusion.attention.backends.sdpa",
|
||||
"class": "SDPABackend",
|
||||
},
|
||||
"SAGE_ATTN": {
|
||||
"module": "vllm_omni.diffusion.attention.backends.sage_attn",
|
||||
"class": "SageAttentionBackend",
|
||||
},
|
||||
"ASCEND": {"module": "vllm_omni.diffusion.attention.backends.ascend_attn", "class": "AscendAttentionBackend"},
|
||||
}
|
||||
|
||||
_BACKENDS_SUPPORT_ATTENTION_MASK = ["SDPA", "ASCEND", "FLASH_ATTN"]
|
||||
def _load_backend_cls(cls_path: str) -> type[AttentionBackend]:
|
||||
"""Load a backend class from its fully qualified path.
|
||||
|
||||
Args:
|
||||
cls_path: Fully qualified class path (e.g.,
|
||||
"vllm_omni.diffusion.attention.backends.sdpa.SDPABackend")
|
||||
|
||||
def load_backend(backend_name: str) -> type[AttentionBackend]:
|
||||
config = _BACKEND_CONFIG[backend_name]
|
||||
|
||||
Returns:
|
||||
The loaded backend class
|
||||
"""
|
||||
module_path, class_name = cls_path.rsplit(".", 1)
|
||||
try:
|
||||
module = importlib.import_module(config["module"])
|
||||
backend_class = getattr(module, config["class"])
|
||||
module = importlib.import_module(module_path)
|
||||
backend_class = getattr(module, class_name)
|
||||
return backend_class
|
||||
except ImportError as e:
|
||||
raise ImportError(f"Failed to import module {config['module']}: {e}")
|
||||
raise ImportError(f"Failed to import module {module_path}: {e}")
|
||||
except AttributeError as e:
|
||||
raise AttributeError(f"Class {config['class']} not found in module: {e}")
|
||||
raise AttributeError(f"Class {class_name} not found in module: {e}")
|
||||
|
||||
|
||||
@cache
|
||||
@@ -54,65 +56,30 @@ def get_attn_backend(head_size: int) -> type[AttentionBackend]:
|
||||
"""
|
||||
Get attention backend for diffusion models.
|
||||
|
||||
The backend is selected based on the following priority:
|
||||
1. DIFFUSION_ATTENTION_BACKEND environment variable (if set, e.g. export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN)
|
||||
2. Default backend (SDPA)
|
||||
The backend selection is delegated to the current platform
|
||||
(vllm_omni.platforms.current_omni_platform), which selects the
|
||||
appropriate backend based on:
|
||||
1. User override via DIFFUSION_ATTENTION_BACKEND environment variable
|
||||
2. Platform-specific defaults and capabilities
|
||||
|
||||
This is similar to how vLLM's get_attn_backend_cls works, where the
|
||||
platform layer decides which backend to use based on hardware capabilities.
|
||||
|
||||
Args:
|
||||
head_size: Head size (currently not used for selection, but kept for API compatibility)
|
||||
head_size: Head size for attention computation (may affect backend selection)
|
||||
|
||||
Returns:
|
||||
The selected attention backend class
|
||||
"""
|
||||
# Check environment variable
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
backend_name = os.environ.get("DIFFUSION_ATTENTION_BACKEND", None)
|
||||
# Check environment variable for user override
|
||||
selected_backend = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
|
||||
|
||||
if detect_device_type() == "cuda" and not is_rocm():
|
||||
compute_capability = torch.cuda.get_device_capability()
|
||||
major, minor = compute_capability
|
||||
if 80 <= major * 10 + minor < 100:
|
||||
if backend_name is None:
|
||||
backend_name = "FLASH_ATTN"
|
||||
else:
|
||||
if backend_name == "FLASH_ATTN":
|
||||
logger.warning(
|
||||
"""Flash Attention requires GPU with compute capability >= 8.0 or < 10.0. "
|
||||
"Falling back to TORCH_SDPA backend."""
|
||||
)
|
||||
backend_name = "TORCH_SDPA"
|
||||
elif detect_device_type() == "cuda" and is_rocm():
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
# Delegate to platform for backend selection
|
||||
backend_cls_path = current_omni_platform.get_diffusion_attn_backend_cls(
|
||||
selected_backend=selected_backend,
|
||||
head_size=head_size,
|
||||
)
|
||||
|
||||
compute_capability = torch.cuda.get_device_capability()
|
||||
major, minor = compute_capability
|
||||
|
||||
# Choose to enable this by default on ROCm
|
||||
# Whenever possible as it is the fastest backend
|
||||
# is_aiter_found_and_supported() checks if aiter library is found
|
||||
# and is aiter supported on the current platform
|
||||
# aiter currently only is supported on gfx942 and gfx950
|
||||
# https://github.com/vllm-project/vllm/blob/main/vllm/_aiter_ops.py
|
||||
if is_aiter_found_and_supported() and 90 < major * 10 + minor < 100:
|
||||
if backend_name is None:
|
||||
backend_name = "FLASH_ATTN"
|
||||
else:
|
||||
if backend_name == "FLASH_ATTN":
|
||||
logger.warning(
|
||||
"Flash Attention requires `aiter` library which is only supported "
|
||||
"on gfx942 and gfx950. Falling back to TORCH_SDPA backend."
|
||||
)
|
||||
backend_name = "TORCH_SDPA"
|
||||
|
||||
if backend_name is not None:
|
||||
backend_name_upper = backend_name.upper()
|
||||
if backend_name_upper not in _BACKEND_CONFIG:
|
||||
valid_backends = list(_BACKEND_CONFIG.keys())
|
||||
raise ValueError(
|
||||
f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}"
|
||||
)
|
||||
logger.info(f"Using attention backend '{backend_name_upper}' for diffusion")
|
||||
return load_backend(backend_name_upper)
|
||||
|
||||
# Default to SDPA
|
||||
return SDPABackend
|
||||
return _load_backend_cls(backend_cls_path)
|
||||
@@ -8,6 +8,8 @@ import torch
|
||||
import torch.distributed as dist
|
||||
from torch import Tensor
|
||||
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
__all__ = ["all_to_all_4D", "all_to_all_5D", "SeqAllToAll4D", "SeqAllToAll5D", "RingComm"]
|
||||
|
||||
|
||||
@@ -48,7 +50,7 @@ def all_to_all_4D(
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
current_omni_platform.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
# if scattering the seq-dim, transpose the heads back to the original dimension
|
||||
@@ -83,7 +85,7 @@ def all_to_all_4D(
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
current_omni_platform.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
|
||||
@@ -155,7 +157,7 @@ def all_to_all_5D(
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
current_omni_platform.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
|
||||
@@ -190,7 +192,7 @@ def all_to_all_5D(
|
||||
if seq_world_size > 1:
|
||||
dist.all_to_all_single(output, input_t, group=group)
|
||||
if use_sync:
|
||||
torch.cuda.synchronize()
|
||||
current_omni_platform.synchronize()
|
||||
else:
|
||||
output = input_t
|
||||
|
||||
|
||||
@@ -9,18 +9,14 @@ from typing import Any
|
||||
|
||||
import torch
|
||||
import torch.distributed
|
||||
from torch.cuda import synchronize
|
||||
from torch.distributed import Backend, ProcessGroup
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion import envs
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
if envs._is_npu():
|
||||
logger.info("torch.npu synchronize")
|
||||
from torch.npu import synchronize
|
||||
|
||||
|
||||
TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
|
||||
|
||||
@@ -124,7 +120,7 @@ class GroupCoordinator:
|
||||
assert self.cpu_group is not None
|
||||
assert self.device_group is not None
|
||||
|
||||
self.device = envs.get_device(local_rank)
|
||||
self.device = current_omni_platform.get_torch_device(local_rank)
|
||||
|
||||
@property
|
||||
def first_rank(self):
|
||||
@@ -618,7 +614,7 @@ class PipelineGroupCoordinator(GroupCoordinator):
|
||||
assert self.cpu_group is not None
|
||||
assert self.device_group is not None
|
||||
|
||||
self.device = envs.get_device(local_rank)
|
||||
self.device = current_omni_platform.get_torch_device(local_rank)
|
||||
|
||||
self.recv_buffer_set: bool = False
|
||||
self.recv_tasks_queue: list[tuple[str, int]] = []
|
||||
@@ -768,7 +764,7 @@ class PipelineGroupCoordinator(GroupCoordinator):
|
||||
|
||||
# To protect against race condition when using batch_isend_irecv().
|
||||
# should take this out once the bug with batch_isend_irecv is resolved.
|
||||
synchronize()
|
||||
current_omni_platform.synchronize()
|
||||
|
||||
ops = []
|
||||
recv_prev_shape_tensor = None
|
||||
@@ -799,7 +795,7 @@ class PipelineGroupCoordinator(GroupCoordinator):
|
||||
for req in reqs:
|
||||
req.wait()
|
||||
|
||||
synchronize()
|
||||
current_omni_platform.synchronize()
|
||||
|
||||
recv_prev_shape = [0, 0, 0]
|
||||
if recv_prev_shape_tensor is not None:
|
||||
|
||||
@@ -35,6 +35,7 @@ from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from vllm_omni.diffusion import envs
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
from .group_coordinator import (
|
||||
GroupCoordinator,
|
||||
@@ -42,14 +43,6 @@ from .group_coordinator import (
|
||||
SequenceParallelGroupCoordinator,
|
||||
)
|
||||
|
||||
if envs._is_npu():
|
||||
from torch.npu import device_count, set_device
|
||||
elif envs._is_musa():
|
||||
from torch_musa.core.device import device_count, set_device
|
||||
else:
|
||||
from torch.cuda import device_count, set_device
|
||||
|
||||
|
||||
env_info = envs.PACKAGES_CHECKER.get_packages_info()
|
||||
|
||||
HAS_FLASH_ATTN = env_info["has_flash_attn"]
|
||||
@@ -396,7 +389,7 @@ def init_distributed_environment(
|
||||
backend: str | None = None,
|
||||
):
|
||||
if backend is None:
|
||||
backend = envs.get_torch_distributed_backend()
|
||||
backend = current_omni_platform.dist_backend
|
||||
logger.debug(
|
||||
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
|
||||
world_size,
|
||||
@@ -416,7 +409,8 @@ def init_distributed_environment(
|
||||
world_size=world_size,
|
||||
rank=rank,
|
||||
)
|
||||
set_device(torch.distributed.get_rank() % device_count())
|
||||
device_id = torch.distributed.get_rank() % current_omni_platform.get_device_count()
|
||||
current_omni_platform.set_device(current_omni_platform.get_torch_device(device_id))
|
||||
# set the local rank
|
||||
# local_rank is not available in torch ProcessGroup,
|
||||
# see https://github.com/pytorch/pytorch/issues/122816
|
||||
@@ -572,7 +566,7 @@ def initialize_model_parallel(
|
||||
backend: str | None = None,
|
||||
) -> None:
|
||||
if backend is None:
|
||||
backend = envs.get_torch_distributed_backend()
|
||||
backend = current_omni_platform.dist_backend
|
||||
"""
|
||||
Initialize model parallel groups.
|
||||
|
||||
@@ -632,7 +626,7 @@ def initialize_model_parallel(
|
||||
|
||||
# FIXME: Since the async p2p communication operation of NPU is not same as cuda in torch,
|
||||
# the pipefusion is not ready for npu yet
|
||||
if envs._is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
assert pipeline_parallel_size == 1, "Current pipefusion is not ready for NPU"
|
||||
|
||||
dit_parallel_size = (
|
||||
|
||||
@@ -5,11 +5,10 @@ import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.utils.platform_utils import detect_device_type
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
def get_local_device() -> torch.device:
|
||||
"""Return the torch device for the current rank based on detected device type."""
|
||||
device_type = detect_device_type()
|
||||
local_rank = os.environ.get("LOCAL_RANK", 0)
|
||||
return torch.device(f"{device_type}:{local_rank}")
|
||||
local_rank = int(os.environ.get("LOCAL_RANK", 0))
|
||||
return current_omni_platform.get_torch_device(local_rank)
|
||||
+22
-141
@@ -5,20 +5,15 @@ import os
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
from packaging import version
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
if TYPE_CHECKING:
|
||||
MASTER_ADDR: str = ""
|
||||
MASTER_PORT: int | None = None
|
||||
CUDA_HOME: str | None = None
|
||||
LOCAL_RANK: int = 0
|
||||
CUDA_VISIBLE_DEVICES: str | None = None
|
||||
CUDA_VERSION: version.Version
|
||||
TORCH_VERSION: version.Version
|
||||
|
||||
environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# ================== Runtime Env Vars ==================
|
||||
@@ -32,127 +27,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
# local rank of the process in the distributed setting, used to determine
|
||||
# the GPU device id
|
||||
"LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")),
|
||||
# used to control the visible devices in the distributed setting
|
||||
"CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
|
||||
}
|
||||
|
||||
|
||||
def _is_hip():
|
||||
has_rocm = torch.version.hip is not None
|
||||
return has_rocm
|
||||
|
||||
|
||||
def _is_cuda():
|
||||
has_cuda = torch.version.cuda is not None
|
||||
return has_cuda
|
||||
|
||||
|
||||
def _is_musa():
|
||||
try:
|
||||
if hasattr(torch, "musa") and torch.musa.is_available():
|
||||
return True
|
||||
except ModuleNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def _is_mps():
|
||||
return torch.backends.mps.is_available()
|
||||
|
||||
|
||||
def _is_npu():
|
||||
try:
|
||||
if hasattr(torch, "npu") and torch.npu.is_available():
|
||||
return True
|
||||
except ModuleNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def get_device(local_rank: int) -> torch.device:
|
||||
if _is_cuda() or _is_hip():
|
||||
return torch.device("cuda", local_rank)
|
||||
elif _is_musa():
|
||||
return torch.device("musa", local_rank)
|
||||
elif _is_mps():
|
||||
return torch.device("mps")
|
||||
elif _is_npu():
|
||||
return torch.device("npu", local_rank)
|
||||
else:
|
||||
return torch.device("cpu")
|
||||
|
||||
|
||||
def get_device_name() -> str:
|
||||
if _is_cuda() or _is_hip():
|
||||
return "cuda"
|
||||
elif _is_musa():
|
||||
return "musa"
|
||||
elif _is_mps():
|
||||
return "mps"
|
||||
elif _is_npu():
|
||||
return "npu"
|
||||
else:
|
||||
return "cpu"
|
||||
|
||||
|
||||
def get_device_version():
|
||||
if _is_hip():
|
||||
hip_version = torch.version.hip
|
||||
hip_version = hip_version.split("-")[0]
|
||||
return hip_version
|
||||
elif _is_cuda():
|
||||
return torch.version.cuda
|
||||
elif _is_musa():
|
||||
return torch.version.musa
|
||||
elif _is_mps():
|
||||
return None
|
||||
elif _is_npu():
|
||||
return None
|
||||
else:
|
||||
raise NotImplementedError("No Accelerators(AMD/NV/MTT GPU, AMD MI instinct accelerators) available")
|
||||
|
||||
|
||||
def get_torch_distributed_backend() -> str:
|
||||
if _is_cuda() or _is_hip():
|
||||
return "nccl"
|
||||
elif _is_musa():
|
||||
return "mccl"
|
||||
elif _is_mps():
|
||||
return "gloo"
|
||||
elif _is_npu():
|
||||
return "hccl"
|
||||
else:
|
||||
raise NotImplementedError("No Accelerators(AMD/NV/MTT GPU, AMD MI instinct accelerators) available")
|
||||
|
||||
|
||||
variables: dict[str, Callable[[], Any]] = {
|
||||
# ================== Other Vars ==================
|
||||
# used in version checking
|
||||
"CUDA_VERSION": lambda: version.parse(get_device_version() or "0.0"),
|
||||
"TORCH_VERSION": lambda: version.parse(version.parse(torch.__version__).base_version),
|
||||
}
|
||||
|
||||
|
||||
def _setup_musa(environment_variables, variables):
|
||||
musa = getattr(torch, "musa", None)
|
||||
if musa is None:
|
||||
return
|
||||
try:
|
||||
if musa.is_available():
|
||||
environment_variables["MUSA_HOME"] = lambda: os.environ.get("MUSA_HOME", None)
|
||||
environment_variables["MUSA_VISIBLE_DEVICES"] = lambda: os.environ.get("MUSA_VISIBLE_DEVICES", None)
|
||||
musa_ver = getattr(getattr(torch, "version", None), "musa", None)
|
||||
if musa_ver:
|
||||
variables["MUSA_VERSION"] = lambda: version.parse(musa_ver)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
try:
|
||||
_setup_musa(environment_variables, variables)
|
||||
except (AttributeError, ModuleNotFoundError):
|
||||
pass
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class PackagesEnvChecker:
|
||||
"""Singleton class for checking package availability."""
|
||||
|
||||
_instance = None
|
||||
|
||||
def __new__(cls):
|
||||
@@ -163,38 +45,39 @@ class PackagesEnvChecker:
|
||||
|
||||
def initialize(self):
|
||||
packages_info = {}
|
||||
packages_info["has_flash_attn"] = self.check_flash_attn(packages_info)
|
||||
packages_info["has_flash_attn"] = self._check_flash_attn(packages_info)
|
||||
self.packages_info = packages_info
|
||||
|
||||
def check_flash_attn(self, packages_info):
|
||||
if not torch.cuda.is_available():
|
||||
def _check_flash_attn(self, packages_info) -> bool:
|
||||
"""Check if flash attention is available and compatible."""
|
||||
platform = current_omni_platform
|
||||
|
||||
# Flash attention requires CUDA-like platforms (CUDA or ROCm)
|
||||
if not platform.is_cuda_alike():
|
||||
return False
|
||||
|
||||
# Check if torch_npu is available
|
||||
if _is_npu():
|
||||
logger.info("`flash_attn` is not ready on torch_npu for now")
|
||||
# Check if devices are available
|
||||
if platform.get_device_count() == 0:
|
||||
return False
|
||||
|
||||
if _is_musa():
|
||||
logger.info("Flash Attention library is not supported on MUSA for the moment.")
|
||||
return False
|
||||
try:
|
||||
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
||||
gpu_name = torch.cuda.get_device_name(device)
|
||||
gpu_name = platform.get_device_name()
|
||||
# Turing/Tesla/T4 GPUs don't support flash attention well
|
||||
if "Turing" in gpu_name or "Tesla" in gpu_name or "T4" in gpu_name:
|
||||
return False
|
||||
else:
|
||||
from flash_attn import __version__
|
||||
|
||||
if __version__ < "2.6.0":
|
||||
raise ImportError("install flash_attn >= 2.6.0")
|
||||
return True
|
||||
from flash_attn import __version__
|
||||
|
||||
if __version__ < "2.6.0":
|
||||
raise ImportError("install flash_attn >= 2.6.0")
|
||||
return True
|
||||
except ImportError:
|
||||
if not packages_info.get("has_aiter", False):
|
||||
logger.warning('Flash Attention library "flash_attn" not found, using pytorch attention implementation')
|
||||
return False
|
||||
|
||||
def get_packages_info(self):
|
||||
def get_packages_info(self) -> dict:
|
||||
"""Get the packages info dictionary."""
|
||||
return self.packages_info
|
||||
|
||||
|
||||
@@ -205,8 +88,6 @@ def __getattr__(name):
|
||||
# lazy evaluation of environment variables
|
||||
if name in environment_variables:
|
||||
return environment_variables[name]()
|
||||
if name in variables:
|
||||
return variables[name]()
|
||||
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, DiffusionOutput
|
||||
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.scheduler import Scheduler
|
||||
from vllm_omni.utils.platform_utils import get_diffusion_worker_class
|
||||
from vllm_omni.diffusion.worker import WorkerProc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -77,9 +77,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
mp.set_start_method("spawn", force=True)
|
||||
processes = []
|
||||
|
||||
# Get the appropriate worker class for current device
|
||||
worker_proc = get_diffusion_worker_class()
|
||||
|
||||
# Launch all worker processes
|
||||
scheduler_pipe_readers = []
|
||||
scheduler_pipe_writers = []
|
||||
@@ -88,7 +85,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
|
||||
reader, writer = mp.Pipe(duplex=False)
|
||||
scheduler_pipe_writers.append(writer)
|
||||
process = mp.Process(
|
||||
target=worker_proc.worker_main,
|
||||
target=WorkerProc.worker_main,
|
||||
args=(
|
||||
i, # rank
|
||||
od_config,
|
||||
|
||||
@@ -335,10 +335,7 @@ class SequenceParallelSplitHook(ModelHook):
|
||||
2. Creates an attention mask indicating valid vs padding positions
|
||||
3. Stores the mask and padding info in ForwardContext
|
||||
"""
|
||||
from vllm_omni.diffusion.attention.selector import (
|
||||
_BACKENDS_SUPPORT_ATTENTION_MASK,
|
||||
get_attn_backend,
|
||||
)
|
||||
from vllm_omni.diffusion.attention.selector import get_attn_backend
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_ring_parallel_world_size,
|
||||
get_sequence_parallel_rank,
|
||||
@@ -359,7 +356,7 @@ class SequenceParallelSplitHook(ModelHook):
|
||||
|
||||
# Check backend compatibility
|
||||
attn_backend = get_attn_backend(-1)
|
||||
if attn_backend.get_name() not in _BACKENDS_SUPPORT_ATTENTION_MASK:
|
||||
if not attn_backend.supports_attention_mask:
|
||||
raise ValueError(
|
||||
f"Sequence length ({seq_len}) is not divisible by SP world size ({world_size}). "
|
||||
f"Cannot use {attn_backend.get_name()} which does not support attention_mask. "
|
||||
|
||||
@@ -91,6 +91,14 @@ class AdaLayerNorm(CustomOp):
|
||||
|
||||
return output, gate_result
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
mod_params: torch.Tensor,
|
||||
index: torch.Tensor = None,
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(x, mod_params, index)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
|
||||
@@ -3,7 +3,7 @@ from typing import Any
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu, is_rocm
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
class CustomOp(nn.Module):
|
||||
@@ -14,16 +14,17 @@ class CustomOp(nn.Module):
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.is_cuda = detect_device_type() == "cuda"
|
||||
self._forward_method = self.dispatch_forward()
|
||||
|
||||
def dispatch_forward(self) -> Callable:
|
||||
if is_rocm():
|
||||
if current_omni_platform.is_rocm():
|
||||
return self.forward_hip
|
||||
elif self.is_cuda:
|
||||
elif current_omni_platform.is_cuda():
|
||||
return self.forward_cuda
|
||||
elif is_npu():
|
||||
elif current_omni_platform.is_npu():
|
||||
return self.forward_npu
|
||||
elif current_omni_platform.is_xpu():
|
||||
return self.forward_xpu
|
||||
else:
|
||||
return self.forward_native
|
||||
|
||||
@@ -44,6 +45,9 @@ class CustomOp(nn.Module):
|
||||
def forward_npu(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_xpu(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_hip(self, *args, **kwargs):
|
||||
# By default, we assume that HIP ops are compatible with CUDA ops.
|
||||
return self.forward_cuda(*args, **kwargs)
|
||||
@@ -137,6 +137,14 @@ class RotaryEmbedding(CustomOp):
|
||||
else:
|
||||
return self.forward_native(x, cos, sin)
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(x, cos, sin)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
|
||||
@@ -24,7 +24,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
get_sp_group,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import get_forward_context
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -610,7 +610,7 @@ class LongCatImageTransformer2DModel(nn.Module):
|
||||
|
||||
ids = torch.cat((txt_ids, img_ids), dim=0)
|
||||
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
freqs_cos, freqs_sin = self.pos_embed(ids.cpu())
|
||||
image_rotary_emb = (freqs_cos.npu(), freqs_sin.npu())
|
||||
else:
|
||||
|
||||
@@ -2,14 +2,14 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Worker classes for diffusion models."""
|
||||
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_worker import (
|
||||
GPUDiffusionWorker,
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.diffusion_worker import (
|
||||
DiffusionWorker,
|
||||
WorkerProc,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"GPUDiffusionModelRunner",
|
||||
"GPUDiffusionWorker",
|
||||
"DiffusionModelRunner",
|
||||
"DiffusionWorker",
|
||||
"WorkerProc",
|
||||
]
|
||||
+1
-1
@@ -34,7 +34,7 @@ from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GPUDiffusionModelRunner:
|
||||
class DiffusionModelRunner:
|
||||
"""
|
||||
Model runner that handles model loading and execution for diffusion models.
|
||||
|
||||
+18
-12
@@ -4,7 +4,7 @@
|
||||
Diffusion Worker for vLLM-Omni.
|
||||
|
||||
Handles GPU infrastructure initialization and delegates model operations
|
||||
to GPUDiffusionModelRunner.
|
||||
to DiffusionModelRunner.
|
||||
"""
|
||||
|
||||
import multiprocessing as mp
|
||||
@@ -31,13 +31,14 @@ from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
|
||||
from vllm_omni.diffusion.profiler import CurrentProfiler
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner
|
||||
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
|
||||
from vllm_omni.lora.request import LoRARequest
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class GPUDiffusionWorker:
|
||||
class DiffusionWorker:
|
||||
"""
|
||||
A worker that manages GPU infrastructure and delegates to the model runner.
|
||||
|
||||
@@ -47,7 +48,7 @@ class GPUDiffusionWorker:
|
||||
- Memory management (sleep/wake)
|
||||
|
||||
All model-related operations (loading, compilation, execution) are
|
||||
delegated to GPUDiffusionModelRunner.
|
||||
delegated to DiffusionModelRunner.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -61,7 +62,7 @@ class GPUDiffusionWorker:
|
||||
self.od_config = od_config
|
||||
self.device: torch.device | None = None
|
||||
self.vllm_config: VllmConfig | None = None
|
||||
self.model_runner: GPUDiffusionModelRunner | None = None
|
||||
self.model_runner: DiffusionModelRunner | None = None
|
||||
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
|
||||
self.lora_manager: DiffusionLoRAManager | None = None
|
||||
self.init_device()
|
||||
@@ -79,8 +80,8 @@ class GPUDiffusionWorker:
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
|
||||
# Setup device
|
||||
self.device = torch.device(f"cuda:{rank}")
|
||||
torch.cuda.set_device(self.device)
|
||||
self.device = current_omni_platform.get_torch_device(rank)
|
||||
current_omni_platform.set_device(self.device)
|
||||
|
||||
# Create vllm_config for parallel configuration
|
||||
vllm_config = VllmConfig()
|
||||
@@ -105,7 +106,7 @@ class GPUDiffusionWorker:
|
||||
)
|
||||
|
||||
# Create model runner and load model
|
||||
self.model_runner = GPUDiffusionModelRunner(
|
||||
self.model_runner = DiffusionModelRunner(
|
||||
vllm_config=self.vllm_config,
|
||||
od_config=self.od_config,
|
||||
device=self.device,
|
||||
@@ -176,7 +177,7 @@ class GPUDiffusionWorker:
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
|
||||
free_bytes_before_sleep = current_omni_platform.get_free_memory()
|
||||
|
||||
# Save the buffers before level 2 sleep
|
||||
if level == 2 and self.model_runner is not None:
|
||||
@@ -185,7 +186,9 @@ class GPUDiffusionWorker:
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
free_bytes_after_sleep = current_omni_platform.get_free_memory()
|
||||
device_id = self.device.index if self.device.index is not None else 0
|
||||
total = current_omni_platform.get_device_total_memory(device_id)
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
used_bytes = total - free_bytes_after_sleep
|
||||
assert freed_bytes >= 0, "Memory usage increased after sleeping."
|
||||
@@ -270,9 +273,9 @@ class WorkerProc:
|
||||
self.gpu_id = gpu_id
|
||||
self._running = True
|
||||
|
||||
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> GPUDiffusionWorker:
|
||||
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> DiffusionWorker:
|
||||
"""Create a worker instance. Override in subclasses for different worker types."""
|
||||
return GPUDiffusionWorker(
|
||||
return DiffusionWorker(
|
||||
local_rank=gpu_id,
|
||||
rank=gpu_id,
|
||||
od_config=od_config,
|
||||
@@ -379,6 +382,9 @@ class WorkerProc:
|
||||
broadcast_handle,
|
||||
) -> None:
|
||||
"""Worker initialization and execution loops."""
|
||||
from vllm_omni.plugins import load_omni_general_plugins
|
||||
|
||||
load_omni_general_plugins()
|
||||
worker_proc = WorkerProc(
|
||||
od_config,
|
||||
gpu_id=rank,
|
||||
@@ -1,7 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NPU worker classes for diffusion models."""
|
||||
|
||||
from vllm_omni.diffusion.worker.npu.npu_worker import NPUWorker, NPUWorkerProc
|
||||
|
||||
__all__ = ["NPUWorker", "NPUWorkerProc"]
|
||||
@@ -1,255 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
|
||||
import torch
|
||||
from vllm.config import LoadConfig, VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes
|
||||
|
||||
from vllm_omni.diffusion.cache.selector import get_cache_backend
|
||||
from vllm_omni.diffusion.data import (
|
||||
DiffusionOutput,
|
||||
OmniDiffusionConfig,
|
||||
)
|
||||
from vllm_omni.diffusion.distributed.parallel_state import (
|
||||
destroy_distributed_env,
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm_omni.diffusion.forward_context import set_forward_context
|
||||
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
|
||||
from vllm_omni.diffusion.request import OmniDiffusionRequest
|
||||
from vllm_omni.diffusion.worker.gpu_diffusion_worker import WorkerProc
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class NPUWorker:
|
||||
"""
|
||||
A worker that executes the model on a single NPU.
|
||||
Inherits from GPUWorker and overrides device-specific initialization.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
local_rank: int,
|
||||
rank: int,
|
||||
od_config: OmniDiffusionConfig,
|
||||
):
|
||||
self.local_rank = local_rank
|
||||
self.rank = rank
|
||||
self.od_config = od_config
|
||||
self.pipeline = None
|
||||
self.device = None
|
||||
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
|
||||
self.init_device_and_model()
|
||||
|
||||
def init_device_and_model(self) -> None:
|
||||
"""Initialize the NPU device and load the model."""
|
||||
world_size = self.od_config.num_gpus
|
||||
rank = self.rank
|
||||
# Set environment variables for distributed initialization
|
||||
os.environ["MASTER_ADDR"] = "localhost"
|
||||
os.environ["MASTER_PORT"] = str(self.od_config.master_port)
|
||||
os.environ["LOCAL_RANK"] = str(self.local_rank)
|
||||
os.environ["RANK"] = str(rank)
|
||||
os.environ["WORLD_SIZE"] = str(world_size)
|
||||
|
||||
device = torch.device(f"npu:{rank}")
|
||||
torch.npu.set_device(device)
|
||||
|
||||
# hack
|
||||
vllm_config = VllmConfig()
|
||||
vllm_config.parallel_config.tensor_parallel_size = self.od_config.parallel_config.tensor_parallel_size
|
||||
vllm_config.parallel_config.data_parallel_size = self.od_config.parallel_config.data_parallel_size
|
||||
self.vllm_config = vllm_config
|
||||
with set_forward_context(vllm_config=vllm_config, omni_diffusion_config=self.od_config):
|
||||
init_distributed_environment(world_size=world_size, rank=rank)
|
||||
logger.info(f"Worker {self.rank}: Initialized device and distributed environment.")
|
||||
parallel_config = self.od_config.parallel_config
|
||||
initialize_model_parallel(
|
||||
data_parallel_size=parallel_config.data_parallel_size,
|
||||
cfg_parallel_size=parallel_config.cfg_parallel_size,
|
||||
sequence_parallel_size=parallel_config.sequence_parallel_size,
|
||||
ulysses_degree=parallel_config.ulysses_degree,
|
||||
ring_degree=parallel_config.ring_degree,
|
||||
tensor_parallel_size=parallel_config.tensor_parallel_size,
|
||||
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
|
||||
)
|
||||
|
||||
load_config = LoadConfig()
|
||||
model_loader = DiffusersPipelineLoader(load_config)
|
||||
time_before_load = time.perf_counter()
|
||||
with DeviceMemoryProfiler() as m:
|
||||
self.pipeline = model_loader.load_model(
|
||||
od_config=self.od_config,
|
||||
load_device=f"npu:{rank}",
|
||||
)
|
||||
time_after_load = time.perf_counter()
|
||||
|
||||
logger.info(
|
||||
"Model loading took %.4f GiB and %.6f seconds",
|
||||
m.consumed_memory / GiB_bytes,
|
||||
time_after_load - time_before_load,
|
||||
)
|
||||
logger.info(f"Worker {self.rank}: Model loaded successfully.")
|
||||
|
||||
# Setup cache backend based on type (both backends use enable()/reset() interface)
|
||||
self.cache_backend = get_cache_backend(self.od_config.cache_backend, self.od_config.cache_config)
|
||||
|
||||
if self.cache_backend is not None:
|
||||
self.cache_backend.enable(self.pipeline)
|
||||
|
||||
def generate(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput:
|
||||
"""
|
||||
Generate output for the given requests.
|
||||
|
||||
Args:
|
||||
requests: List of diffusion requests
|
||||
|
||||
Returns:
|
||||
DiffusionOutput with generated results
|
||||
"""
|
||||
return self.execute_model(requests, self.od_config)
|
||||
|
||||
@torch.inference_mode()
|
||||
def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput:
|
||||
"""
|
||||
Execute a forward pass.
|
||||
"""
|
||||
assert self.pipeline is not None
|
||||
if not reqs or len(reqs) == 0:
|
||||
raise ValueError("Cannot execute model with empty request list")
|
||||
# TODO: dealing with first req for now
|
||||
req = reqs[0]
|
||||
|
||||
if req.generator is None and req.seed is not None:
|
||||
req.generator = torch.Generator(device=self.device).manual_seed(req.seed)
|
||||
|
||||
# Refresh cache context if needed
|
||||
if (
|
||||
not getattr(req, "skip_cache_refresh", False)
|
||||
and self.cache_backend is not None
|
||||
and self.cache_backend.is_enabled()
|
||||
):
|
||||
self.cache_backend.refresh(self.pipeline, req.num_inference_steps)
|
||||
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config):
|
||||
output = self.pipeline.forward(req)
|
||||
return output
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
return self.pipeline.load_weights(weights)
|
||||
|
||||
def sleep(self, level: int = 1) -> bool:
|
||||
"""
|
||||
Put the worker to sleep. The worker should not process any requests.
|
||||
The caller should guarantee that no requests are being processed
|
||||
during the sleep period, before `wake_up` is called.
|
||||
|
||||
Args:
|
||||
level: The sleep level. Level 1 sleep will offload the model
|
||||
weights and discard the kv cache.
|
||||
Currently only support level 1.
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
|
||||
|
||||
# Save the buffers before level 2 sleep
|
||||
if level == 2:
|
||||
model = self.pipeline
|
||||
self._sleep_saved_buffers = {name: buffer.cpu().clone() for name, buffer in model.named_buffers()}
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
|
||||
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
|
||||
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
|
||||
used_bytes = total - free_bytes_after_sleep
|
||||
assert freed_bytes >= 0, "Memory usage increased after sleeping."
|
||||
logger.info(
|
||||
"Sleep mode freed %.2f GiB memory, %.2f GiB memory is still in use.",
|
||||
freed_bytes / GiB_bytes,
|
||||
used_bytes / GiB_bytes,
|
||||
)
|
||||
return True
|
||||
|
||||
def wake_up(self, tags: list[str] | None = None) -> bool:
|
||||
"""
|
||||
Wake up the worker from sleep mode. See the sleep function
|
||||
method for more details.
|
||||
|
||||
Args:
|
||||
tags: An optional list of tags to reallocate the worker memory
|
||||
for specific memory allocations. Values must be in
|
||||
`("weights")`. If None, all memory is reallocated.
|
||||
wake_up should be called with all tags (or None) before the
|
||||
worker is used again.
|
||||
"""
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
allocator.wake_up(tags)
|
||||
|
||||
# Restore the buffers after level 2 sleep
|
||||
if len(self._sleep_saved_buffers):
|
||||
model = self.pipeline
|
||||
for name, buffer in model.named_buffers():
|
||||
if name in self._sleep_saved_buffers:
|
||||
buffer.data.copy_(self._sleep_saved_buffers[name].data)
|
||||
self._sleep_saved_buffers = {}
|
||||
return True
|
||||
|
||||
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
|
||||
if self.od_config.enable_sleep_mode:
|
||||
from vllm.device_allocator.cumem import CuMemAllocator
|
||||
|
||||
allocator = CuMemAllocator.get_instance()
|
||||
if tag == "weights":
|
||||
assert allocator.get_current_usage() == 0, "Sleep mode can only be used for one instance per process."
|
||||
return allocator.use_memory_pool(tag=tag)
|
||||
else:
|
||||
return nullcontext()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
destroy_distributed_env()
|
||||
|
||||
|
||||
class NPUWorkerProc(WorkerProc):
|
||||
"""Wrapper that runs one NPUWorker in a separate process."""
|
||||
|
||||
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> NPUWorker:
|
||||
"""Create an NPUWorker instead of GPUWorker."""
|
||||
return NPUWorker(
|
||||
local_rank=gpu_id,
|
||||
rank=gpu_id,
|
||||
od_config=od_config,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def worker_main(
|
||||
rank: int,
|
||||
od_config: OmniDiffusionConfig,
|
||||
pipe_writer: mp.connection.Connection,
|
||||
broadcast_handle,
|
||||
) -> None:
|
||||
"""Worker initialization and execution loops."""
|
||||
|
||||
worker_proc = NPUWorkerProc(
|
||||
od_config,
|
||||
gpu_id=rank,
|
||||
broadcast_handle=broadcast_handle,
|
||||
)
|
||||
logger.info(f"Worker {rank}: Scheduler loop started.")
|
||||
pipe_writer.send(
|
||||
{
|
||||
"status": "ready",
|
||||
"result_handle": worker_proc.result_mq_handle if rank == 0 else None,
|
||||
}
|
||||
)
|
||||
worker_proc.worker_busy_loop()
|
||||
logger.info(f"Worker {rank}: Shutdown complete.")
|
||||
Reference in new issue
Block a user