[Hardware] Support platforms and plugin system (#774)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
b11d436e19
commit
c9392dff01
103 files changed
+1770
-1218
No files matched your search
@@ -91,6 +91,14 @@ class AdaLayerNorm(CustomOp):
|
||||
|
||||
return output, gate_result
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
mod_params: torch.Tensor,
|
||||
index: torch.Tensor = None,
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(x, mod_params, index)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
|
||||
@@ -3,7 +3,7 @@ from typing import Any
|
||||
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm_omni.utils.platform_utils import detect_device_type, is_npu, is_rocm
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
class CustomOp(nn.Module):
|
||||
@@ -14,16 +14,17 @@ class CustomOp(nn.Module):
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.is_cuda = detect_device_type() == "cuda"
|
||||
self._forward_method = self.dispatch_forward()
|
||||
|
||||
def dispatch_forward(self) -> Callable:
|
||||
if is_rocm():
|
||||
if current_omni_platform.is_rocm():
|
||||
return self.forward_hip
|
||||
elif self.is_cuda:
|
||||
elif current_omni_platform.is_cuda():
|
||||
return self.forward_cuda
|
||||
elif is_npu():
|
||||
elif current_omni_platform.is_npu():
|
||||
return self.forward_npu
|
||||
elif current_omni_platform.is_xpu():
|
||||
return self.forward_xpu
|
||||
else:
|
||||
return self.forward_native
|
||||
|
||||
@@ -44,6 +45,9 @@ class CustomOp(nn.Module):
|
||||
def forward_npu(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_xpu(self, *args, **kwargs):
|
||||
raise NotImplementedError
|
||||
|
||||
def forward_hip(self, *args, **kwargs):
|
||||
# By default, we assume that HIP ops are compatible with CUDA ops.
|
||||
return self.forward_cuda(*args, **kwargs)
|
||||
@@ -137,6 +137,14 @@ class RotaryEmbedding(CustomOp):
|
||||
else:
|
||||
return self.forward_native(x, cos, sin)
|
||||
|
||||
def forward_xpu(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
cos: torch.Tensor,
|
||||
sin: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return self.forward_native(x, cos, sin)
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
|
||||
Reference in new issue
Block a user