[Feat] Add NPU Backend support for vLLM-Omni (#89)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
88c00fb271
commit
ec9e04027d
25 files changed
+2932
-85
No files matched your search
@@ -5,7 +5,11 @@ import os
|
||||
|
||||
import torch
|
||||
|
||||
from vllm_omni.utils.platform_utils import detect_device_type
|
||||
|
||||
|
||||
def get_local_device() -> torch.device:
|
||||
"""Return the torch device for the current rank."""
|
||||
return torch.device(f"cuda:{os.environ.get('LOCAL_RANK', 0)}")
|
||||
"""Return the torch device for the current rank based on detected device type."""
|
||||
device_type = detect_device_type()
|
||||
local_rank = os.environ.get("LOCAL_RANK", 0)
|
||||
return torch.device(f"{device_type}:{local_rank}")
|
||||
Reference in new issue
Block a user