[Hardware] Support platforms and plugin system (#774)

Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
Canlin Guo authored and GitHub committed 2026-01-28 22:12:52 +08:00
1 parent b11d436e19
commit c9392dff01
103 files changed
+1770 -1218

No files matched your search

@@ -7,12 +7,18 @@ from typing import Generic, TypeVar
import torch
from vllm_omni.platforms import current_omni_platform
class AttentionBackend(ABC):
"""Abstract class for diffusion attention backends."""
accept_output_buffer: bool = False
@classmethod
def supports_attention_mask(cls) -> bool:
return False
@staticmethod
@abstractmethod
def get_name() -> str:
@@ -77,12 +83,58 @@ class AttentionImpl(ABC, Generic[T]):
) -> None:
raise NotImplementedError
@abstractmethod
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: T,
attn_metadata: T | None = None,
) -> torch.Tensor:
"""Dispatch to platform-specific forward implementation."""
if current_omni_platform.is_rocm():
return self.forward_hip(query, key, value, attn_metadata)
elif current_omni_platform.is_cuda():
return self.forward_cuda(query, key, value, attn_metadata)
elif current_omni_platform.is_npu():
return self.forward_npu(query, key, value, attn_metadata)
elif current_omni_platform.is_xpu():
return self.forward_xpu(query, key, value, attn_metadata)
else:
raise NotImplementedError(f"No forward implementation for platform: {current_omni_platform}")
def forward_cuda(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: T | None = None,
) -> torch.Tensor:
raise NotImplementedError
def forward_npu(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: T | None = None,
) -> torch.Tensor:
raise NotImplementedError
def forward_xpu(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: T | None = None,
) -> torch.Tensor:
raise NotImplementedError
def forward_hip(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: T | None = None,
) -> torch.Tensor:
# By default, HIP ops are compatible with CUDA ops.
return self.forward_cuda(query, key, value, attn_metadata)
@@ -1,92 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from importlib.util import find_spec
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionBackend,
AttentionImpl,
AttentionMetadata,
)
logger = init_logger(__name__)
class AscendAttentionBackend(AttentionBackend):
accept_output_buffer: bool = True
@staticmethod
def get_supported_head_sizes() -> list[int]:
return [x for x in range(1024)] # todo
@staticmethod
def get_name() -> str:
return "ASCEND"
@staticmethod
def get_impl_cls() -> type["AscendAttentionBackendImpl"]:
return AscendAttentionBackendImpl
class AscendAttentionBackendImpl(AttentionImpl):
def __init__(
self,
num_heads: int,
head_size: int,
softmax_scale: float,
causal: bool = False,
num_kv_heads: int | None = None,
prefix: str = "",
**extra_impl_args,
) -> None:
self.causal = causal
self.softmax_scale = softmax_scale
def forward_native(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
query, key, value = (x.permute(0, 2, 1, 3) for x in (query, key, value))
attention_mask = attn_metadata.attn_mask if attn_metadata else None
output = torch.nn.functional.scaled_dot_product_attention(
query,
key,
value,
attn_mask=attention_mask,
dropout_p=0.0,
is_causal=self.causal,
scale=self.softmax_scale,
)
out = output.permute(0, 2, 1, 3)
return out
def forward(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
if find_spec("mindiesd"):
from mindiesd import attention_forward
attention_mask = attn_metadata.attn_mask if attn_metadata else None
output = attention_forward(
query,
key,
value,
attn_mask=attention_mask,
opt_mode="manual",
op_type="fused_attn_score",
layout="BNSD",
)
return output
else:
return self.forward_native(query, key, value, attn_metadata)
@@ -9,13 +9,6 @@ from vllm_omni.diffusion.attention.backends.abstract import (
AttentionImpl,
AttentionMetadata,
)
from vllm_omni.diffusion.attention.backends.utils.fa import (
_pad_input,
_unpad_input,
_upad_input,
flash_attn_func,
flash_attn_varlen_func,
)
logger = init_logger(__name__)
@@ -23,6 +16,10 @@ logger = init_logger(__name__)
class FlashAttentionBackend(AttentionBackend):
accept_output_buffer: bool = True
@classmethod
def supports_attention_mask(cls) -> bool:
return True
@staticmethod
def get_supported_head_sizes() -> list[int]:
return [64, 96, 128, 192, 256]
@@ -51,25 +48,22 @@ class FlashAttentionImpl(AttentionImpl):
self.causal = causal
self.softmax_scale = softmax_scale
def forward(
def forward_cuda(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
"""
Flash attention implementation.
"""CUDA/ROCm flash attention implementation."""
from vllm_omni.diffusion.attention.backends.utils.fa import (
_pad_input,
_unpad_input,
_upad_input,
flash_attn_func,
flash_attn_varlen_func,
)
Args:
query: (batch_size, seq_len, num_heads, head_dim)
key: (batch_size, seq_len, num_heads, head_dim)
value: (batch_size, seq_len, num_heads, head_dim)
attn_metadata: AttentionMetadata. Attention mask is supported as attn_metadata.attn_mask
Returns:
(batch_size, seq_len, num_heads, head_dim)
"""
query_length = query.size(1)
attention_mask = attn_metadata.attn_mask if attn_metadata is not None else None
# Contains at least one padding token in the sequence
@@ -106,3 +100,25 @@ class FlashAttentionImpl(AttentionImpl):
softmax_scale=self.softmax_scale,
)
return out
def forward_npu(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: AttentionMetadata = None,
) -> torch.Tensor:
"""NPU attention implementation using mindiesd."""
from mindiesd import attention_forward
attention_mask = attn_metadata.attn_mask if attn_metadata else None
output = attention_forward(
query,
key,
value,
attn_mask=attention_mask,
opt_mode="manual",
op_type="fused_attn_score",
layout="BNSD",
)
return output
@@ -0,0 +1,152 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Diffusion attention backend registry.
This module provides an enum-based registry for diffusion attention backends,
similar to vLLM's AttentionBackendEnum. Each backend registers its class path,
and platforms can override or extend backends using register_backend().
"""
from collections.abc import Callable
from enum import Enum, EnumMeta
from typing import TYPE_CHECKING
from vllm.logger import init_logger
from vllm.utils.import_utils import resolve_obj_by_qualname
if TYPE_CHECKING:
from vllm_omni.diffusion.attention.backends.abstract import AttentionBackend
logger = init_logger(__name__)
class _DiffusionBackendEnumMeta(EnumMeta):
"""Metaclass for DiffusionAttentionBackendEnum to provide better error messages."""
def __getitem__(cls, name: str) -> "DiffusionAttentionBackendEnum":
"""Get backend by name with helpful error messages."""
try:
return super().__getitem__(name) # type: ignore[return-value]
except KeyError:
members = list(cls.__members__.keys())
valid_backends = ", ".join(members)
raise ValueError(
f"Unknown diffusion attention backend: '{name}'. Valid options are: {valid_backends}"
) from None
class DiffusionAttentionBackendEnum(Enum, metaclass=_DiffusionBackendEnumMeta):
"""Enumeration of all supported diffusion attention backends.
The enum value is the default class path, but this can be overridden
at runtime using register_backend().
To get the actual backend class (respecting overrides), use:
backend.get_class()
Example:
# Get backend class
backend = DiffusionAttentionBackendEnum.FLASH_ATTN
backend_cls = backend.get_class()
# Register custom backend
@register_diffusion_backend(DiffusionAttentionBackendEnum.CUSTOM)
class MyCustomBackend:
...
"""
# Common backends (available on most platforms)
FLASH_ATTN = "vllm_omni.diffusion.attention.backends.flash_attn.FlashAttentionBackend"
TORCH_SDPA = "vllm_omni.diffusion.attention.backends.sdpa.SDPABackend"
SAGE_ATTN = "vllm_omni.diffusion.attention.backends.sage_attn.SageAttentionBackend"
def get_path(self, include_classname: bool = True) -> str:
"""Get the class path for this backend (respects overrides).
Returns:
The fully qualified class path string
Raises:
ValueError: If backend has empty path and is not registered
"""
path = _DIFFUSION_ATTN_OVERRIDES.get(self, self.value)
if not path:
raise ValueError(
f"Backend {self.name} must be registered before use. "
f"Use register_diffusion_backend(DiffusionAttentionBackendEnum.{self.name}, "
f"'your.module.YourClass')"
)
if not include_classname:
path = path.rsplit(".", 1)[0]
return path
def get_class(self) -> "type[AttentionBackend]":
"""Get the backend class (respects overrides).
Returns:
The backend class
Raises:
ImportError: If the backend class cannot be imported
ValueError: If backend has empty path and is not registered
"""
return resolve_obj_by_qualname(self.get_path())
def is_overridden(self) -> bool:
"""Check if this backend has been overridden.
Returns:
True if the backend has a registered override
"""
return self in _DIFFUSION_ATTN_OVERRIDES
def clear_override(self) -> None:
"""Clear any override for this backend, reverting to the default."""
_DIFFUSION_ATTN_OVERRIDES.pop(self, None)
# Override registry
_DIFFUSION_ATTN_OVERRIDES: dict[DiffusionAttentionBackendEnum, str] = {}
def register_diffusion_backend(
backend: DiffusionAttentionBackendEnum,
class_path: str | None = None,
) -> Callable[[type], type]:
"""Register or override a diffusion backend implementation.
Args:
backend: The DiffusionAttentionBackendEnum member to register
class_path: Optional class path. If not provided and used as
decorator, will be auto-generated from the class.
Returns:
Decorator function if class_path is None, otherwise a no-op
Examples:
# Override an existing backend
@register_diffusion_backend(DiffusionAttentionBackendEnum.FLASH_ATTN)
class MyCustomFlashAttn:
...
# Override an existing backend (e.g., ASCEND_ATTN)
@register_diffusion_backend(DiffusionAttentionBackendEnum.ASCEND_ATTN)
class CustomAscendAttentionBackend:
...
# Direct registration
register_diffusion_backend(
DiffusionAttentionBackendEnum.CUSTOM,
"my.module.MyCustomBackend"
)
"""
def decorator(cls: type) -> type:
_DIFFUSION_ATTN_OVERRIDES[backend] = f"{cls.__module__}.{cls.__qualname__}"
return cls
if class_path is not None:
_DIFFUSION_ATTN_OVERRIDES[backend] = class_path
return lambda x: x
return decorator
@@ -54,7 +54,7 @@ class SageAttentionImpl(AttentionImpl):
self.causal = causal
self.softmax_scale = softmax_scale
def forward(
def forward_cuda(
self,
query: torch.Tensor,
key: torch.Tensor,
@@ -16,6 +16,10 @@ logger = init_logger(__name__)
class SDPABackend(AttentionBackend):
accept_output_buffer: bool = True
@classmethod
def supports_attention_mask(cls) -> bool:
return True
@staticmethod
def get_supported_head_sizes() -> list[int]:
return [x for x in range(1024)] # todo
@@ -15,9 +15,9 @@
import torch
import torch.nn.functional as F
from vllm_omni.utils.platform_utils import is_rocm
from vllm_omni.platforms import current_omni_platform
if is_rocm():
if current_omni_platform.is_rocm():
from vllm._aiter_ops import is_aiter_found_and_supported
# Choose to enable this by default on ROCm
+47 -80
View File
@@ -1,52 +1,54 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
Diffusion attention backend selector.
This module provides the interface for selecting diffusion attention backends.
The actual backend selection logic is delegated to the platform layer
(vllm_omni.platforms), similar to how vLLM handles attention backend selection.
Usage:
from vllm_omni.diffusion.attention.selector import get_attn_backend
# Get the appropriate backend for current platform
backend_cls = get_attn_backend(head_size=64)
# Or override via environment variable
# export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN
"""
import importlib
import os
from functools import cache
import torch
from vllm.logger import init_logger
from vllm_omni.diffusion.attention.backends.abstract import (
AttentionBackend,
)
from vllm_omni.diffusion.attention.backends.sdpa import SDPABackend
from vllm_omni.utils.platform_utils import detect_device_type, is_rocm
logger = init_logger(__name__)
# environment variable value -> backend module and class
_BACKEND_CONFIG = {
"FLASH_ATTN": {
"module": "vllm_omni.diffusion.attention.backends.flash_attn",
"class": "FlashAttentionBackend",
},
"TORCH_SDPA": {
"module": "vllm_omni.diffusion.attention.backends.sdpa",
"class": "SDPABackend",
},
"SAGE_ATTN": {
"module": "vllm_omni.diffusion.attention.backends.sage_attn",
"class": "SageAttentionBackend",
},
"ASCEND": {"module": "vllm_omni.diffusion.attention.backends.ascend_attn", "class": "AscendAttentionBackend"},
}
_BACKENDS_SUPPORT_ATTENTION_MASK = ["SDPA", "ASCEND", "FLASH_ATTN"]
def _load_backend_cls(cls_path: str) -> type[AttentionBackend]:
"""Load a backend class from its fully qualified path.
Args:
cls_path: Fully qualified class path (e.g.,
"vllm_omni.diffusion.attention.backends.sdpa.SDPABackend")
def load_backend(backend_name: str) -> type[AttentionBackend]:
config = _BACKEND_CONFIG[backend_name]
Returns:
The loaded backend class
"""
module_path, class_name = cls_path.rsplit(".", 1)
try:
module = importlib.import_module(config["module"])
backend_class = getattr(module, config["class"])
module = importlib.import_module(module_path)
backend_class = getattr(module, class_name)
return backend_class
except ImportError as e:
raise ImportError(f"Failed to import module {config['module']}: {e}")
raise ImportError(f"Failed to import module {module_path}: {e}")
except AttributeError as e:
raise AttributeError(f"Class {config['class']} not found in module: {e}")
raise AttributeError(f"Class {class_name} not found in module: {e}")
@cache
@@ -54,65 +56,30 @@ def get_attn_backend(head_size: int) -> type[AttentionBackend]:
"""
Get attention backend for diffusion models.
The backend is selected based on the following priority:
1. DIFFUSION_ATTENTION_BACKEND environment variable (if set, e.g. export DIFFUSION_ATTENTION_BACKEND=FLASH_ATTN)
2. Default backend (SDPA)
The backend selection is delegated to the current platform
(vllm_omni.platforms.current_omni_platform), which selects the
appropriate backend based on:
1. User override via DIFFUSION_ATTENTION_BACKEND environment variable
2. Platform-specific defaults and capabilities
This is similar to how vLLM's get_attn_backend_cls works, where the
platform layer decides which backend to use based on hardware capabilities.
Args:
head_size: Head size (currently not used for selection, but kept for API compatibility)
head_size: Head size for attention computation (may affect backend selection)
Returns:
The selected attention backend class
"""
# Check environment variable
from vllm_omni.platforms import current_omni_platform
backend_name = os.environ.get("DIFFUSION_ATTENTION_BACKEND", None)
# Check environment variable for user override
selected_backend = os.environ.get("DIFFUSION_ATTENTION_BACKEND")
if detect_device_type() == "cuda" and not is_rocm():
compute_capability = torch.cuda.get_device_capability()
major, minor = compute_capability
if 80 <= major * 10 + minor < 100:
if backend_name is None:
backend_name = "FLASH_ATTN"
else:
if backend_name == "FLASH_ATTN":
logger.warning(
"""Flash Attention requires GPU with compute capability >= 8.0 or < 10.0. "
"Falling back to TORCH_SDPA backend."""
)
backend_name = "TORCH_SDPA"
elif detect_device_type() == "cuda" and is_rocm():
from vllm._aiter_ops import is_aiter_found_and_supported
# Delegate to platform for backend selection
backend_cls_path = current_omni_platform.get_diffusion_attn_backend_cls(
selected_backend=selected_backend,
head_size=head_size,
)
compute_capability = torch.cuda.get_device_capability()
major, minor = compute_capability
# Choose to enable this by default on ROCm
# Whenever possible as it is the fastest backend
# is_aiter_found_and_supported() checks if aiter library is found
# and is aiter supported on the current platform
# aiter currently only is supported on gfx942 and gfx950
# https://github.com/vllm-project/vllm/blob/main/vllm/_aiter_ops.py
if is_aiter_found_and_supported() and 90 < major * 10 + minor < 100:
if backend_name is None:
backend_name = "FLASH_ATTN"
else:
if backend_name == "FLASH_ATTN":
logger.warning(
"Flash Attention requires `aiter` library which is only supported "
"on gfx942 and gfx950. Falling back to TORCH_SDPA backend."
)
backend_name = "TORCH_SDPA"
if backend_name is not None:
backend_name_upper = backend_name.upper()
if backend_name_upper not in _BACKEND_CONFIG:
valid_backends = list(_BACKEND_CONFIG.keys())
raise ValueError(
f"Invalid attention backend for diffusion: '{backend_name}'. Valid backends are: {valid_backends}"
)
logger.info(f"Using attention backend '{backend_name_upper}' for diffusion")
return load_backend(backend_name_upper)
# Default to SDPA
return SDPABackend
return _load_backend_cls(backend_cls_path)
+6 -4
View File
@@ -8,6 +8,8 @@ import torch
import torch.distributed as dist
from torch import Tensor
from vllm_omni.platforms import current_omni_platform
__all__ = ["all_to_all_4D", "all_to_all_5D", "SeqAllToAll4D", "SeqAllToAll5D", "RingComm"]
@@ -48,7 +50,7 @@ def all_to_all_4D(
if seq_world_size > 1:
dist.all_to_all_single(output, input_t, group=group)
if use_sync:
torch.cuda.synchronize()
current_omni_platform.synchronize()
else:
output = input_t
# if scattering the seq-dim, transpose the heads back to the original dimension
@@ -83,7 +85,7 @@ def all_to_all_4D(
if seq_world_size > 1:
dist.all_to_all_single(output, input_t, group=group)
if use_sync:
torch.cuda.synchronize()
current_omni_platform.synchronize()
else:
output = input_t
@@ -155,7 +157,7 @@ def all_to_all_5D(
if seq_world_size > 1:
dist.all_to_all_single(output, input_t, group=group)
if use_sync:
torch.cuda.synchronize()
current_omni_platform.synchronize()
else:
output = input_t
@@ -190,7 +192,7 @@ def all_to_all_5D(
if seq_world_size > 1:
dist.all_to_all_single(output, input_t, group=group)
if use_sync:
torch.cuda.synchronize()
current_omni_platform.synchronize()
else:
output = input_t
@@ -9,18 +9,14 @@ from typing import Any
import torch
import torch.distributed
from torch.cuda import synchronize
from torch.distributed import Backend, ProcessGroup
from vllm.logger import init_logger
from vllm_omni.diffusion import envs
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
if envs._is_npu():
logger.info("torch.npu synchronize")
from torch.npu import synchronize
TensorMetadata = namedtuple("TensorMetadata", ["device", "dtype", "size"])
@@ -124,7 +120,7 @@ class GroupCoordinator:
assert self.cpu_group is not None
assert self.device_group is not None
self.device = envs.get_device(local_rank)
self.device = current_omni_platform.get_torch_device(local_rank)
@property
def first_rank(self):
@@ -618,7 +614,7 @@ class PipelineGroupCoordinator(GroupCoordinator):
assert self.cpu_group is not None
assert self.device_group is not None
self.device = envs.get_device(local_rank)
self.device = current_omni_platform.get_torch_device(local_rank)
self.recv_buffer_set: bool = False
self.recv_tasks_queue: list[tuple[str, int]] = []
@@ -768,7 +764,7 @@ class PipelineGroupCoordinator(GroupCoordinator):
# To protect against race condition when using batch_isend_irecv().
# should take this out once the bug with batch_isend_irecv is resolved.
synchronize()
current_omni_platform.synchronize()
ops = []
recv_prev_shape_tensor = None
@@ -799,7 +795,7 @@ class PipelineGroupCoordinator(GroupCoordinator):
for req in reqs:
req.wait()
synchronize()
current_omni_platform.synchronize()
recv_prev_shape = [0, 0, 0]
if recv_prev_shape_tensor is not None:
@@ -35,6 +35,7 @@ from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
from vllm.logger import init_logger
from vllm_omni.diffusion import envs
from vllm_omni.platforms import current_omni_platform
from .group_coordinator import (
GroupCoordinator,
@@ -42,14 +43,6 @@ from .group_coordinator import (
SequenceParallelGroupCoordinator,
)
if envs._is_npu():
from torch.npu import device_count, set_device
elif envs._is_musa():
from torch_musa.core.device import device_count, set_device
else:
from torch.cuda import device_count, set_device
env_info = envs.PACKAGES_CHECKER.get_packages_info()
HAS_FLASH_ATTN = env_info["has_flash_attn"]
@@ -396,7 +389,7 @@ def init_distributed_environment(
backend: str | None = None,
):
if backend is None:
backend = envs.get_torch_distributed_backend()
backend = current_omni_platform.dist_backend
logger.debug(
"world_size=%d rank=%d local_rank=%d distributed_init_method=%s backend=%s",
world_size,
@@ -416,7 +409,8 @@ def init_distributed_environment(
world_size=world_size,
rank=rank,
)
set_device(torch.distributed.get_rank() % device_count())
device_id = torch.distributed.get_rank() % current_omni_platform.get_device_count()
current_omni_platform.set_device(current_omni_platform.get_torch_device(device_id))
# set the local rank
# local_rank is not available in torch ProcessGroup,
# see https://github.com/pytorch/pytorch/issues/122816
@@ -572,7 +566,7 @@ def initialize_model_parallel(
backend: str | None = None,
) -> None:
if backend is None:
backend = envs.get_torch_distributed_backend()
backend = current_omni_platform.dist_backend
"""
Initialize model parallel groups.
@@ -632,7 +626,7 @@ def initialize_model_parallel(
# FIXME: Since the async p2p communication operation of NPU is not same as cuda in torch,
# the pipefusion is not ready for npu yet
if envs._is_npu():
if current_omni_platform.is_npu():
assert pipeline_parallel_size == 1, "Current pipefusion is not ready for NPU"
dit_parallel_size = (
+3 -4
View File
@@ -5,11 +5,10 @@ import os
import torch
from vllm_omni.utils.platform_utils import detect_device_type
from vllm_omni.platforms import current_omni_platform
def get_local_device() -> torch.device:
"""Return the torch device for the current rank based on detected device type."""
device_type = detect_device_type()
local_rank = os.environ.get("LOCAL_RANK", 0)
return torch.device(f"{device_type}:{local_rank}")
local_rank = int(os.environ.get("LOCAL_RANK", 0))
return current_omni_platform.get_torch_device(local_rank)
+22 -141
View File
@@ -5,20 +5,15 @@ import os
from collections.abc import Callable
from typing import TYPE_CHECKING, Any
import torch
from packaging import version
from vllm.logger import init_logger
logger = init_logger(__name__)
from vllm_omni.platforms import current_omni_platform
if TYPE_CHECKING:
MASTER_ADDR: str = ""
MASTER_PORT: int | None = None
CUDA_HOME: str | None = None
LOCAL_RANK: int = 0
CUDA_VISIBLE_DEVICES: str | None = None
CUDA_VERSION: version.Version
TORCH_VERSION: version.Version
environment_variables: dict[str, Callable[[], Any]] = {
# ================== Runtime Env Vars ==================
@@ -32,127 +27,14 @@ environment_variables: dict[str, Callable[[], Any]] = {
# local rank of the process in the distributed setting, used to determine
# the GPU device id
"LOCAL_RANK": lambda: int(os.environ.get("LOCAL_RANK", "0")),
# used to control the visible devices in the distributed setting
"CUDA_VISIBLE_DEVICES": lambda: os.environ.get("CUDA_VISIBLE_DEVICES", None),
}
def _is_hip():
has_rocm = torch.version.hip is not None
return has_rocm
def _is_cuda():
has_cuda = torch.version.cuda is not None
return has_cuda
def _is_musa():
try:
if hasattr(torch, "musa") and torch.musa.is_available():
return True
except ModuleNotFoundError:
return False
def _is_mps():
return torch.backends.mps.is_available()
def _is_npu():
try:
if hasattr(torch, "npu") and torch.npu.is_available():
return True
except ModuleNotFoundError:
return False
def get_device(local_rank: int) -> torch.device:
if _is_cuda() or _is_hip():
return torch.device("cuda", local_rank)
elif _is_musa():
return torch.device("musa", local_rank)
elif _is_mps():
return torch.device("mps")
elif _is_npu():
return torch.device("npu", local_rank)
else:
return torch.device("cpu")
def get_device_name() -> str:
if _is_cuda() or _is_hip():
return "cuda"
elif _is_musa():
return "musa"
elif _is_mps():
return "mps"
elif _is_npu():
return "npu"
else:
return "cpu"
def get_device_version():
if _is_hip():
hip_version = torch.version.hip
hip_version = hip_version.split("-")[0]
return hip_version
elif _is_cuda():
return torch.version.cuda
elif _is_musa():
return torch.version.musa
elif _is_mps():
return None
elif _is_npu():
return None
else:
raise NotImplementedError("No Accelerators(AMD/NV/MTT GPU, AMD MI instinct accelerators) available")
def get_torch_distributed_backend() -> str:
if _is_cuda() or _is_hip():
return "nccl"
elif _is_musa():
return "mccl"
elif _is_mps():
return "gloo"
elif _is_npu():
return "hccl"
else:
raise NotImplementedError("No Accelerators(AMD/NV/MTT GPU, AMD MI instinct accelerators) available")
variables: dict[str, Callable[[], Any]] = {
# ================== Other Vars ==================
# used in version checking
"CUDA_VERSION": lambda: version.parse(get_device_version() or "0.0"),
"TORCH_VERSION": lambda: version.parse(version.parse(torch.__version__).base_version),
}
def _setup_musa(environment_variables, variables):
musa = getattr(torch, "musa", None)
if musa is None:
return
try:
if musa.is_available():
environment_variables["MUSA_HOME"] = lambda: os.environ.get("MUSA_HOME", None)
environment_variables["MUSA_VISIBLE_DEVICES"] = lambda: os.environ.get("MUSA_VISIBLE_DEVICES", None)
musa_ver = getattr(getattr(torch, "version", None), "musa", None)
if musa_ver:
variables["MUSA_VERSION"] = lambda: version.parse(musa_ver)
except Exception:
pass
try:
_setup_musa(environment_variables, variables)
except (AttributeError, ModuleNotFoundError):
pass
logger = init_logger(__name__)
class PackagesEnvChecker:
"""Singleton class for checking package availability."""
_instance = None
def __new__(cls):
@@ -163,38 +45,39 @@ class PackagesEnvChecker:
def initialize(self):
packages_info = {}
packages_info["has_flash_attn"] = self.check_flash_attn(packages_info)
packages_info["has_flash_attn"] = self._check_flash_attn(packages_info)
self.packages_info = packages_info
def check_flash_attn(self, packages_info):
if not torch.cuda.is_available():
def _check_flash_attn(self, packages_info) -> bool:
"""Check if flash attention is available and compatible."""
platform = current_omni_platform
# Flash attention requires CUDA-like platforms (CUDA or ROCm)
if not platform.is_cuda_alike():
return False
# Check if torch_npu is available
if _is_npu():
logger.info("`flash_attn` is not ready on torch_npu for now")
# Check if devices are available
if platform.get_device_count() == 0:
return False
if _is_musa():
logger.info("Flash Attention library is not supported on MUSA for the moment.")
return False
try:
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
gpu_name = torch.cuda.get_device_name(device)
gpu_name = platform.get_device_name()
# Turing/Tesla/T4 GPUs don't support flash attention well
if "Turing" in gpu_name or "Tesla" in gpu_name or "T4" in gpu_name:
return False
else:
from flash_attn import __version__
if __version__ < "2.6.0":
raise ImportError("install flash_attn >= 2.6.0")
return True
from flash_attn import __version__
if __version__ < "2.6.0":
raise ImportError("install flash_attn >= 2.6.0")
return True
except ImportError:
if not packages_info.get("has_aiter", False):
logger.warning('Flash Attention library "flash_attn" not found, using pytorch attention implementation')
return False
def get_packages_info(self):
def get_packages_info(self) -> dict:
"""Get the packages info dictionary."""
return self.packages_info
@@ -205,8 +88,6 @@ def __getattr__(name):
# lazy evaluation of environment variables
if name in environment_variables:
return environment_variables[name]()
if name in variables:
return variables[name]()
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
@@ -10,7 +10,7 @@ from vllm_omni.diffusion.data import SHUTDOWN_MESSAGE, DiffusionOutput
from vllm_omni.diffusion.executor.abstract import DiffusionExecutor
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.scheduler import Scheduler
from vllm_omni.utils.platform_utils import get_diffusion_worker_class
from vllm_omni.diffusion.worker import WorkerProc
logger = init_logger(__name__)
@@ -77,9 +77,6 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
mp.set_start_method("spawn", force=True)
processes = []
# Get the appropriate worker class for current device
worker_proc = get_diffusion_worker_class()
# Launch all worker processes
scheduler_pipe_readers = []
scheduler_pipe_writers = []
@@ -88,7 +85,7 @@ class MultiprocDiffusionExecutor(DiffusionExecutor):
reader, writer = mp.Pipe(duplex=False)
scheduler_pipe_writers.append(writer)
process = mp.Process(
target=worker_proc.worker_main,
target=WorkerProc.worker_main,
args=(
i, # rank
od_config,
@@ -335,10 +335,7 @@ class SequenceParallelSplitHook(ModelHook):
2. Creates an attention mask indicating valid vs padding positions
3. Stores the mask and padding info in ForwardContext
"""
from vllm_omni.diffusion.attention.selector import (
_BACKENDS_SUPPORT_ATTENTION_MASK,
get_attn_backend,
)
from vllm_omni.diffusion.attention.selector import get_attn_backend
from vllm_omni.diffusion.distributed.parallel_state import (
get_ring_parallel_world_size,
get_sequence_parallel_rank,
@@ -359,7 +356,7 @@ class SequenceParallelSplitHook(ModelHook):
# Check backend compatibility
attn_backend = get_attn_backend(-1)
if attn_backend.get_name() not in _BACKENDS_SUPPORT_ATTENTION_MASK:
if not attn_backend.supports_attention_mask:
raise ValueError(
f"Sequence length ({seq_len}) is not divisible by SP world size ({world_size}). "
f"Cannot use {attn_backend.get_name()} which does not support attention_mask. "
@@ -91,6 +91,14 @@ class AdaLayerNorm(CustomOp):
return output, gate_result
def forward_xpu(
self,
x: torch.Tensor,
mod_params: torch.Tensor,
index: torch.Tensor = None,
) -> torch.Tensor:
return self.forward_native(x, mod_params, index)
def forward_native(
self,
x: torch.Tensor,
+9 -5
View File
@@ -3,7 +3,7 @@ from typing import Any
import torch.nn as nn
from vllm_omni.utils.platform_utils import detect_device_type, is_npu, is_rocm
from vllm_omni.platforms import current_omni_platform
class CustomOp(nn.Module):
@@ -14,16 +14,17 @@ class CustomOp(nn.Module):
def __init__(self) -> None:
super().__init__()
self.is_cuda = detect_device_type() == "cuda"
self._forward_method = self.dispatch_forward()
def dispatch_forward(self) -> Callable:
if is_rocm():
if current_omni_platform.is_rocm():
return self.forward_hip
elif self.is_cuda:
elif current_omni_platform.is_cuda():
return self.forward_cuda
elif is_npu():
elif current_omni_platform.is_npu():
return self.forward_npu
elif current_omni_platform.is_xpu():
return self.forward_xpu
else:
return self.forward_native
@@ -44,6 +45,9 @@ class CustomOp(nn.Module):
def forward_npu(self, *args, **kwargs):
raise NotImplementedError
def forward_xpu(self, *args, **kwargs):
raise NotImplementedError
def forward_hip(self, *args, **kwargs):
# By default, we assume that HIP ops are compatible with CUDA ops.
return self.forward_cuda(*args, **kwargs)
+8
View File
@@ -137,6 +137,14 @@ class RotaryEmbedding(CustomOp):
else:
return self.forward_native(x, cos, sin)
def forward_xpu(
self,
x: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> torch.Tensor:
return self.forward_native(x, cos, sin)
def forward_native(
self,
x: torch.Tensor,
@@ -24,7 +24,7 @@ from vllm_omni.diffusion.distributed.parallel_state import (
get_sp_group,
)
from vllm_omni.diffusion.forward_context import get_forward_context
from vllm_omni.utils.platform_utils import is_npu
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
@@ -610,7 +610,7 @@ class LongCatImageTransformer2DModel(nn.Module):
ids = torch.cat((txt_ids, img_ids), dim=0)
if is_npu():
if current_omni_platform.is_npu():
freqs_cos, freqs_sin = self.pos_embed(ids.cpu())
image_rotary_emb = (freqs_cos.npu(), freqs_sin.npu())
else:
+5 -5
View File
@@ -2,14 +2,14 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker classes for diffusion models."""
from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner
from vllm_omni.diffusion.worker.gpu_diffusion_worker import (
GPUDiffusionWorker,
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
from vllm_omni.diffusion.worker.diffusion_worker import (
DiffusionWorker,
WorkerProc,
)
__all__ = [
"GPUDiffusionModelRunner",
"GPUDiffusionWorker",
"DiffusionModelRunner",
"DiffusionWorker",
"WorkerProc",
]
@@ -34,7 +34,7 @@ from vllm_omni.distributed.omni_connectors.utils.config import ConnectorSpec
logger = init_logger(__name__)
class GPUDiffusionModelRunner:
class DiffusionModelRunner:
"""
Model runner that handles model loading and execution for diffusion models.
@@ -4,7 +4,7 @@
Diffusion Worker for vLLM-Omni.
Handles GPU infrastructure initialization and delegates model operations
to GPUDiffusionModelRunner.
to DiffusionModelRunner.
"""
import multiprocessing as mp
@@ -31,13 +31,14 @@ from vllm_omni.diffusion.forward_context import set_forward_context
from vllm_omni.diffusion.lora.manager import DiffusionLoRAManager
from vllm_omni.diffusion.profiler import CurrentProfiler
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.gpu_diffusion_model_runner import GPUDiffusionModelRunner
from vllm_omni.diffusion.worker.diffusion_model_runner import DiffusionModelRunner
from vllm_omni.lora.request import LoRARequest
from vllm_omni.platforms import current_omni_platform
logger = init_logger(__name__)
class GPUDiffusionWorker:
class DiffusionWorker:
"""
A worker that manages GPU infrastructure and delegates to the model runner.
@@ -47,7 +48,7 @@ class GPUDiffusionWorker:
- Memory management (sleep/wake)
All model-related operations (loading, compilation, execution) are
delegated to GPUDiffusionModelRunner.
delegated to DiffusionModelRunner.
"""
def __init__(
@@ -61,7 +62,7 @@ class GPUDiffusionWorker:
self.od_config = od_config
self.device: torch.device | None = None
self.vllm_config: VllmConfig | None = None
self.model_runner: GPUDiffusionModelRunner | None = None
self.model_runner: DiffusionModelRunner | None = None
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
self.lora_manager: DiffusionLoRAManager | None = None
self.init_device()
@@ -79,8 +80,8 @@ class GPUDiffusionWorker:
os.environ["WORLD_SIZE"] = str(world_size)
# Setup device
self.device = torch.device(f"cuda:{rank}")
torch.cuda.set_device(self.device)
self.device = current_omni_platform.get_torch_device(rank)
current_omni_platform.set_device(self.device)
# Create vllm_config for parallel configuration
vllm_config = VllmConfig()
@@ -105,7 +106,7 @@ class GPUDiffusionWorker:
)
# Create model runner and load model
self.model_runner = GPUDiffusionModelRunner(
self.model_runner = DiffusionModelRunner(
vllm_config=self.vllm_config,
od_config=self.od_config,
device=self.device,
@@ -176,7 +177,7 @@ class GPUDiffusionWorker:
"""
from vllm.device_allocator.cumem import CuMemAllocator
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
free_bytes_before_sleep = current_omni_platform.get_free_memory()
# Save the buffers before level 2 sleep
if level == 2 and self.model_runner is not None:
@@ -185,7 +186,9 @@ class GPUDiffusionWorker:
allocator = CuMemAllocator.get_instance()
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
free_bytes_after_sleep = current_omni_platform.get_free_memory()
device_id = self.device.index if self.device.index is not None else 0
total = current_omni_platform.get_device_total_memory(device_id)
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
used_bytes = total - free_bytes_after_sleep
assert freed_bytes >= 0, "Memory usage increased after sleeping."
@@ -270,9 +273,9 @@ class WorkerProc:
self.gpu_id = gpu_id
self._running = True
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> GPUDiffusionWorker:
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> DiffusionWorker:
"""Create a worker instance. Override in subclasses for different worker types."""
return GPUDiffusionWorker(
return DiffusionWorker(
local_rank=gpu_id,
rank=gpu_id,
od_config=od_config,
@@ -379,6 +382,9 @@ class WorkerProc:
broadcast_handle,
) -> None:
"""Worker initialization and execution loops."""
from vllm_omni.plugins import load_omni_general_plugins
load_omni_general_plugins()
worker_proc = WorkerProc(
od_config,
gpu_id=rank,
@@ -1,7 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""NPU worker classes for diffusion models."""
from vllm_omni.diffusion.worker.npu.npu_worker import NPUWorker, NPUWorkerProc
__all__ = ["NPUWorker", "NPUWorkerProc"]
@@ -1,255 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import multiprocessing as mp
import os
import time
from collections.abc import Iterable
from contextlib import AbstractContextManager, nullcontext
import torch
from vllm.config import LoadConfig, VllmConfig
from vllm.logger import init_logger
from vllm.utils.mem_utils import DeviceMemoryProfiler, GiB_bytes
from vllm_omni.diffusion.cache.selector import get_cache_backend
from vllm_omni.diffusion.data import (
DiffusionOutput,
OmniDiffusionConfig,
)
from vllm_omni.diffusion.distributed.parallel_state import (
destroy_distributed_env,
init_distributed_environment,
initialize_model_parallel,
)
from vllm_omni.diffusion.forward_context import set_forward_context
from vllm_omni.diffusion.model_loader.diffusers_loader import DiffusersPipelineLoader
from vllm_omni.diffusion.request import OmniDiffusionRequest
from vllm_omni.diffusion.worker.gpu_diffusion_worker import WorkerProc
logger = init_logger(__name__)
class NPUWorker:
"""
A worker that executes the model on a single NPU.
Inherits from GPUWorker and overrides device-specific initialization.
"""
def __init__(
self,
local_rank: int,
rank: int,
od_config: OmniDiffusionConfig,
):
self.local_rank = local_rank
self.rank = rank
self.od_config = od_config
self.pipeline = None
self.device = None
self._sleep_saved_buffers: dict[str, torch.Tensor] = {}
self.init_device_and_model()
def init_device_and_model(self) -> None:
"""Initialize the NPU device and load the model."""
world_size = self.od_config.num_gpus
rank = self.rank
# Set environment variables for distributed initialization
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = str(self.od_config.master_port)
os.environ["LOCAL_RANK"] = str(self.local_rank)
os.environ["RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)
device = torch.device(f"npu:{rank}")
torch.npu.set_device(device)
# hack
vllm_config = VllmConfig()
vllm_config.parallel_config.tensor_parallel_size = self.od_config.parallel_config.tensor_parallel_size
vllm_config.parallel_config.data_parallel_size = self.od_config.parallel_config.data_parallel_size
self.vllm_config = vllm_config
with set_forward_context(vllm_config=vllm_config, omni_diffusion_config=self.od_config):
init_distributed_environment(world_size=world_size, rank=rank)
logger.info(f"Worker {self.rank}: Initialized device and distributed environment.")
parallel_config = self.od_config.parallel_config
initialize_model_parallel(
data_parallel_size=parallel_config.data_parallel_size,
cfg_parallel_size=parallel_config.cfg_parallel_size,
sequence_parallel_size=parallel_config.sequence_parallel_size,
ulysses_degree=parallel_config.ulysses_degree,
ring_degree=parallel_config.ring_degree,
tensor_parallel_size=parallel_config.tensor_parallel_size,
pipeline_parallel_size=parallel_config.pipeline_parallel_size,
)
load_config = LoadConfig()
model_loader = DiffusersPipelineLoader(load_config)
time_before_load = time.perf_counter()
with DeviceMemoryProfiler() as m:
self.pipeline = model_loader.load_model(
od_config=self.od_config,
load_device=f"npu:{rank}",
)
time_after_load = time.perf_counter()
logger.info(
"Model loading took %.4f GiB and %.6f seconds",
m.consumed_memory / GiB_bytes,
time_after_load - time_before_load,
)
logger.info(f"Worker {self.rank}: Model loaded successfully.")
# Setup cache backend based on type (both backends use enable()/reset() interface)
self.cache_backend = get_cache_backend(self.od_config.cache_backend, self.od_config.cache_config)
if self.cache_backend is not None:
self.cache_backend.enable(self.pipeline)
def generate(self, requests: list[OmniDiffusionRequest]) -> DiffusionOutput:
"""
Generate output for the given requests.
Args:
requests: List of diffusion requests
Returns:
DiffusionOutput with generated results
"""
return self.execute_model(requests, self.od_config)
@torch.inference_mode()
def execute_model(self, reqs: list[OmniDiffusionRequest], od_config: OmniDiffusionConfig) -> DiffusionOutput:
"""
Execute a forward pass.
"""
assert self.pipeline is not None
if not reqs or len(reqs) == 0:
raise ValueError("Cannot execute model with empty request list")
# TODO: dealing with first req for now
req = reqs[0]
if req.generator is None and req.seed is not None:
req.generator = torch.Generator(device=self.device).manual_seed(req.seed)
# Refresh cache context if needed
if (
not getattr(req, "skip_cache_refresh", False)
and self.cache_backend is not None
and self.cache_backend.is_enabled()
):
self.cache_backend.refresh(self.pipeline, req.num_inference_steps)
with set_forward_context(vllm_config=self.vllm_config, omni_diffusion_config=self.od_config):
output = self.pipeline.forward(req)
return output
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
return self.pipeline.load_weights(weights)
def sleep(self, level: int = 1) -> bool:
"""
Put the worker to sleep. The worker should not process any requests.
The caller should guarantee that no requests are being processed
during the sleep period, before `wake_up` is called.
Args:
level: The sleep level. Level 1 sleep will offload the model
weights and discard the kv cache.
Currently only support level 1.
"""
from vllm.device_allocator.cumem import CuMemAllocator
free_bytes_before_sleep = torch.cuda.mem_get_info()[0]
# Save the buffers before level 2 sleep
if level == 2:
model = self.pipeline
self._sleep_saved_buffers = {name: buffer.cpu().clone() for name, buffer in model.named_buffers()}
allocator = CuMemAllocator.get_instance()
allocator.sleep(offload_tags=("weights",) if level == 1 else tuple())
free_bytes_after_sleep, total = torch.cuda.mem_get_info()
freed_bytes = free_bytes_after_sleep - free_bytes_before_sleep
used_bytes = total - free_bytes_after_sleep
assert freed_bytes >= 0, "Memory usage increased after sleeping."
logger.info(
"Sleep mode freed %.2f GiB memory, %.2f GiB memory is still in use.",
freed_bytes / GiB_bytes,
used_bytes / GiB_bytes,
)
return True
def wake_up(self, tags: list[str] | None = None) -> bool:
"""
Wake up the worker from sleep mode. See the sleep function
method for more details.
Args:
tags: An optional list of tags to reallocate the worker memory
for specific memory allocations. Values must be in
`("weights")`. If None, all memory is reallocated.
wake_up should be called with all tags (or None) before the
worker is used again.
"""
from vllm.device_allocator.cumem import CuMemAllocator
allocator = CuMemAllocator.get_instance()
allocator.wake_up(tags)
# Restore the buffers after level 2 sleep
if len(self._sleep_saved_buffers):
model = self.pipeline
for name, buffer in model.named_buffers():
if name in self._sleep_saved_buffers:
buffer.data.copy_(self._sleep_saved_buffers[name].data)
self._sleep_saved_buffers = {}
return True
def _maybe_get_memory_pool_context(self, tag: str) -> AbstractContextManager:
if self.od_config.enable_sleep_mode:
from vllm.device_allocator.cumem import CuMemAllocator
allocator = CuMemAllocator.get_instance()
if tag == "weights":
assert allocator.get_current_usage() == 0, "Sleep mode can only be used for one instance per process."
return allocator.use_memory_pool(tag=tag)
else:
return nullcontext()
def shutdown(self) -> None:
destroy_distributed_env()
class NPUWorkerProc(WorkerProc):
"""Wrapper that runs one NPUWorker in a separate process."""
def _create_worker(self, gpu_id: int, od_config: OmniDiffusionConfig) -> NPUWorker:
"""Create an NPUWorker instead of GPUWorker."""
return NPUWorker(
local_rank=gpu_id,
rank=gpu_id,
od_config=od_config,
)
@staticmethod
def worker_main(
rank: int,
od_config: OmniDiffusionConfig,
pipe_writer: mp.connection.Connection,
broadcast_handle,
) -> None:
"""Worker initialization and execution loops."""
worker_proc = NPUWorkerProc(
od_config,
gpu_id=rank,
broadcast_handle=broadcast_handle,
)
logger.info(f"Worker {rank}: Scheduler loop started.")
pipe_writer.send(
{
"status": "ready",
"result_handle": worker_proc.result_mq_handle if rank == 0 else None,
}
)
worker_proc.worker_busy_loop()
logger.info(f"Worker {rank}: Shutdown complete.")