[Feat] Add MUSA platform support for Moore Threads GPUs (#2337)
Signed-off-by: Xiaodong Ye <yeahdongcn@gmail.com>
This commit is contained in:
1 parent
bf5bd0a4c0
commit
3def008b32
15 files changed
+454
-11
No files matched your search
@@ -177,20 +177,24 @@ markers = [
|
||||
"rocm: Tests that run on AMD/ROCm (auto-added)",
|
||||
"xpu: Tests that run on XPU (auto-added)",
|
||||
"npu: Tests that run on NPU/Ascend (auto-added)",
|
||||
"musa: Tests that run on MUSA/Moore Threads (auto-added)",
|
||||
# specified computation resources marks (auto-added)
|
||||
"H100: Tests that require H100 GPU",
|
||||
"L4: Tests that require L4 GPU",
|
||||
"MI325: Tests that require MI325 GPU (AMD/ROCm)",
|
||||
"S5000: Tests that require S5000 GPU (Moore Threads/MUSA)",
|
||||
"A2: Tests that require A2 NPU",
|
||||
"A3: Tests that require A3 NPU",
|
||||
"distributed_cuda: Tests that require multi cards on CUDA platform",
|
||||
"distributed_rocm: Tests that require multi cards on ROCm platform",
|
||||
"distributed_xpu: Tests that require multi cards on XPU platform",
|
||||
"distributed_npu: Tests that require multi cards on NPU platform",
|
||||
"distributed_musa: Tests that require multi cards on MUSA platform",
|
||||
"skipif_cuda: Skip if the num of CUDA cards is less than the required",
|
||||
"skipif_rocm: Skip if the num of ROCm cards is less than the required",
|
||||
"skipif_xpu: Skip if the num of XPU cards is less than the required",
|
||||
"skipif_npu: Skip if the num of NPU cards is less than the required",
|
||||
"skipif_musa: Skip if the num of MUSA cards is less than the required",
|
||||
# more detailed markers
|
||||
"slow: Slow tests (may skip in quick CI)",
|
||||
"benchmark: Benchmark tests",
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
-r common.txt
|
||||
# MUSA platform dependencies
|
||||
torchada>=0.1.46
|
||||
onnxruntime>=1.23.2
|
||||
@@ -3,7 +3,7 @@ Setup script for vLLM-Omni with hardware-dependent installation.
|
||||
|
||||
This setup.py implements platform-aware dependency routing so users can run
|
||||
`pip install vllm-omni` and automatically receive the correct platform-specific
|
||||
dependencies (CUDA/ROCm/CPU/XPU/NPU) without requiring extras like `[cuda]`.
|
||||
dependencies (CUDA/ROCm/CPU/XPU/NPU/MUSA) without requiring extras like `[cuda]`.
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -46,16 +46,16 @@ def detect_target_device() -> str:
|
||||
|
||||
Priority order:
|
||||
1. VLLM_OMNI_TARGET_DEVICE environment variable (highest priority)
|
||||
2. Torch backend detection (cuda, rocm, npu, xpu)
|
||||
2. Torch backend detection (cuda, rocm, npu, xpu, musa)
|
||||
3. CPU fallback (default)
|
||||
|
||||
Returns:
|
||||
str: Device name ('cuda', 'rocm', 'npu', 'xpu', or 'cpu')
|
||||
str: Device name ('cuda', 'rocm', 'npu', 'xpu', 'musa', or 'cpu')
|
||||
"""
|
||||
# Priority 1: Explicit override via environment variable
|
||||
target_device = os.environ.get("VLLM_OMNI_TARGET_DEVICE")
|
||||
if target_device:
|
||||
valid_devices = ["cuda", "rocm", "npu", "xpu", "cpu"]
|
||||
valid_devices = ["cuda", "rocm", "npu", "xpu", "musa", "cpu"]
|
||||
if target_device.lower() in valid_devices:
|
||||
print(f"Using target device from VLLM_OMNI_TARGET_DEVICE: {target_device.lower()}")
|
||||
return target_device.lower()
|
||||
@@ -97,6 +97,15 @@ def detect_target_device() -> str:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Check for MUSA (Moore Threads)
|
||||
if hasattr(torch, "musa"):
|
||||
try:
|
||||
if torch.musa.is_available():
|
||||
print("Detected MUSA backend from torch")
|
||||
return "musa"
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
print("No GPU backend detected in torch, defaulting to CPU")
|
||||
return "cpu"
|
||||
|
||||
@@ -152,6 +161,8 @@ def get_vllm_omni_version() -> str:
|
||||
version += f"{sep}npu"
|
||||
elif device == "xpu":
|
||||
version += f"{sep}xpu"
|
||||
elif device == "musa":
|
||||
version += f"{sep}musa"
|
||||
elif device == "cpu":
|
||||
version += f"{sep}cpu"
|
||||
else:
|
||||
|
||||
+40
-6
@@ -430,13 +430,41 @@ def xpu_marks(*, res: str, num_cards: int):
|
||||
return marks + [test_distributed]
|
||||
|
||||
|
||||
def musa_marks(*, res: str, num_cards: int):
|
||||
"""
|
||||
Get a collection of pytest marks to apply for `@musa_test`.
|
||||
|
||||
Args:
|
||||
res: Resource type, e.g., "S5000".
|
||||
num_cards: Number of GPU cards required.
|
||||
|
||||
Returns:
|
||||
List of pytest marks to apply.
|
||||
"""
|
||||
test_platform_detail = pytest.mark.musa
|
||||
|
||||
if res == "S5000":
|
||||
test_resource = pytest.mark.S5000
|
||||
else:
|
||||
raise ValueError(f"Invalid MUSA resource type: {res}. Supported: S5000")
|
||||
|
||||
marks = [test_resource, test_platform_detail]
|
||||
|
||||
if num_cards == 1:
|
||||
return marks
|
||||
else:
|
||||
test_distributed = pytest.mark.distributed_musa(num_cards=num_cards)
|
||||
# TODO: add MUSA support for `skipif_musa` marker
|
||||
return marks + [test_distributed]
|
||||
|
||||
|
||||
def gpu_marks(*, res: str, num_cards: int):
|
||||
"""
|
||||
Get a collection of pytest marks to apply for `@gpu_test`.
|
||||
Platform is automatically determined based on resource type.
|
||||
|
||||
Args:
|
||||
res: Resource type, e.g., "L4", "H100" for CUDA, or "MI325" for ROCm, or "B60" for XPU.
|
||||
res: Resource type, e.g., "L4", "H100" for CUDA, or "MI325" for ROCm, or "B60" for XPU, or "S5000" for MUSA.
|
||||
num_cards: Number of GPU cards required.
|
||||
|
||||
Returns:
|
||||
@@ -449,7 +477,9 @@ def gpu_marks(*, res: str, num_cards: int):
|
||||
return [test_platform] + rocm_marks(res=res, num_cards=num_cards)
|
||||
if res == "B60":
|
||||
return [test_platform] + xpu_marks(res=res, num_cards=num_cards)
|
||||
raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325")
|
||||
if res == "S5000":
|
||||
return [test_platform] + musa_marks(res=res, num_cards=num_cards)
|
||||
raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325, B60, S5000")
|
||||
|
||||
|
||||
def npu_marks(*, res: str, num_cards: int):
|
||||
@@ -476,13 +506,13 @@ def npu_marks(*, res: str, num_cards: int):
|
||||
def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
"""
|
||||
Get a collection of pytest marks to apply for `@hardware_test`,
|
||||
including CUDA, ROCm, XPU, and NPU,
|
||||
including CUDA, ROCm, XPU, NPU, and MUSA,
|
||||
based on the specified platforms and resources.
|
||||
"""
|
||||
# Validate platforms
|
||||
# Don't validate platform details in this decorator
|
||||
for platform, _ in res.items():
|
||||
if platform not in ("cuda", "rocm", "xpu", "npu"):
|
||||
if platform not in ("cuda", "rocm", "xpu", "npu", "musa"):
|
||||
raise ValueError(f"Unsupported platform: {platform}")
|
||||
|
||||
# Normalize num_cards
|
||||
@@ -505,6 +535,8 @@ def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
cards = num_cards_dict[platform]
|
||||
if platform == "cuda" or platform == "rocm" or platform == "xpu":
|
||||
marks = gpu_marks(res=resource, num_cards=cards)
|
||||
elif platform == "musa":
|
||||
marks = musa_marks(res=resource, num_cards=cards)
|
||||
elif platform == "npu":
|
||||
marks = npu_marks(res=resource, num_cards=cards)
|
||||
else:
|
||||
@@ -522,15 +554,17 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
|
||||
res: Mapping from platform to resource type. Supported platforms/resources:
|
||||
- cuda: L4, H100
|
||||
- rocm: MI325
|
||||
- xpu: B60
|
||||
- npu: A2, A3
|
||||
- musa: S5000
|
||||
num_cards: Number of cards required. Can be:
|
||||
- int: same card count for all platforms (default: 1)
|
||||
- dict: per-platform card count, e.g., {"cuda": 2, "rocm": 2}
|
||||
|
||||
Example:
|
||||
@hardware_test(
|
||||
res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
|
||||
num_cards={"cuda": 2, "rocm": 2, "npu": 2},
|
||||
res={"cuda": "L4", "rocm": "MI325", "npu": "A2", "musa": "S5000"},
|
||||
num_cards={"cuda": 2, "rocm": 2, "npu": 2, "musa": 2},
|
||||
)
|
||||
def test_multi_platform():
|
||||
...
|
||||
|
||||
@@ -99,6 +99,8 @@ class AttentionImpl(ABC, Generic[T]):
|
||||
return self.forward_npu(query, key, value, attn_metadata)
|
||||
elif current_omni_platform.is_xpu():
|
||||
return self.forward_xpu(query, key, value, attn_metadata)
|
||||
elif current_omni_platform.is_musa():
|
||||
return self.forward_musa(query, key, value, attn_metadata)
|
||||
else:
|
||||
raise NotImplementedError(f"No forward implementation for platform: {current_omni_platform}")
|
||||
|
||||
@@ -138,3 +140,13 @@ class AttentionImpl(ABC, Generic[T]):
|
||||
) -> torch.Tensor:
|
||||
# By default, HIP ops are compatible with CUDA ops.
|
||||
return self.forward_cuda(query, key, value, attn_metadata)
|
||||
|
||||
def forward_musa(
|
||||
self,
|
||||
query: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
value: torch.Tensor,
|
||||
attn_metadata: T | None = None,
|
||||
) -> torch.Tensor:
|
||||
# By default, MUSA ops are compatible with CUDA ops.
|
||||
return self.forward_cuda(query, key, value, attn_metadata)
|
||||
@@ -35,6 +35,9 @@ elif current_omni_platform.is_xpu():
|
||||
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func # noqa: F401
|
||||
except (ImportError, ModuleNotFoundError):
|
||||
pass
|
||||
elif current_omni_platform.is_musa():
|
||||
# XXX (MUSA): Add MUSA-specific Flash Attention when available
|
||||
pass
|
||||
else:
|
||||
# CUDA: try FA3 -> FA2 fallback chain
|
||||
# Try FA3 from fa3-fwd PyPI package
|
||||
|
||||
@@ -25,6 +25,8 @@ class CustomOp(nn.Module):
|
||||
return self.forward_npu
|
||||
elif current_omni_platform.is_xpu():
|
||||
return self.forward_xpu
|
||||
elif current_omni_platform.is_musa():
|
||||
return self.forward_musa
|
||||
else:
|
||||
return self.forward_native
|
||||
|
||||
@@ -51,3 +53,7 @@ class CustomOp(nn.Module):
|
||||
def forward_hip(self, *args, **kwargs):
|
||||
# By default, we assume that HIP ops are compatible with CUDA ops.
|
||||
return self.forward_cuda(*args, **kwargs)
|
||||
|
||||
def forward_musa(self, *args, **kwargs):
|
||||
# By default, we assume that MUSA ops are compatible with CUDA ops.
|
||||
return self.forward_cuda(*args, **kwargs)
|
||||
@@ -105,11 +105,28 @@ def xpu_omni_platform_plugin() -> str | None:
|
||||
return "vllm_omni.platforms.xpu.platform.XPUOmniPlatform" if is_xpu else None
|
||||
|
||||
|
||||
def musa_omni_platform_plugin() -> str | None:
|
||||
"""Check if MUSA OmniPlatform should be activated."""
|
||||
is_musa = False
|
||||
logger.debug("Checking if MUSA OmniPlatform is available.")
|
||||
try:
|
||||
import torchada
|
||||
|
||||
if torchada.is_musa_platform():
|
||||
is_musa = True
|
||||
logger.debug("Confirmed MUSA OmniPlatform is available.")
|
||||
except Exception as e:
|
||||
logger.debug("MUSA OmniPlatform is not available because: %s", str(e))
|
||||
|
||||
return "vllm_omni.platforms.musa.platform.MUSAOmniPlatform" if is_musa else None
|
||||
|
||||
|
||||
builtin_omni_platform_plugins = {
|
||||
"cuda": cuda_omni_platform_plugin,
|
||||
"rocm": rocm_omni_platform_plugin,
|
||||
"npu": npu_omni_platform_plugin,
|
||||
"xpu": xpu_omni_platform_plugin,
|
||||
"musa": musa_omni_platform_plugin,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -15,6 +15,7 @@ class OmniPlatformEnum(Enum):
|
||||
ROCM = "rocm"
|
||||
NPU = "npu"
|
||||
XPU = "xpu"
|
||||
MUSA = "musa"
|
||||
UNSPECIFIED = "unspecified"
|
||||
|
||||
|
||||
@@ -41,6 +42,9 @@ class OmniPlatform(Platform):
|
||||
def is_rocm(self) -> bool:
|
||||
return self._omni_enum == OmniPlatformEnum.ROCM
|
||||
|
||||
def is_musa(self) -> bool:
|
||||
return self._omni_enum == OmniPlatformEnum.MUSA
|
||||
|
||||
@classmethod
|
||||
def get_omni_ar_worker_cls(cls) -> str:
|
||||
raise NotImplementedError
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm_omni.platforms.musa.platform import MUSAOmniPlatform
|
||||
|
||||
__all__ = ["MUSAOmniPlatform"]
|
||||
@@ -0,0 +1,123 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm_musa.platform import MUSAPlatformBase
|
||||
|
||||
from vllm_omni.diffusion.attention.backends.registry import DiffusionAttentionBackendEnum
|
||||
from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MUSAOmniPlatform(OmniPlatform, MUSAPlatformBase):
|
||||
"""MUSA/Moore Threads GPU implementation of OmniPlatform.
|
||||
|
||||
Inherits all MUSA-specific implementations from vllm-musa's MUSAPlatformBase,
|
||||
and adds Omni-specific interfaces from OmniPlatform.
|
||||
"""
|
||||
|
||||
_omni_enum = OmniPlatformEnum.MUSA
|
||||
|
||||
@classmethod
|
||||
def get_omni_ar_worker_cls(cls) -> str:
|
||||
return "vllm_omni.platforms.musa.worker.musa_ar_worker.MUSAARWorker"
|
||||
|
||||
@classmethod
|
||||
def get_omni_generation_worker_cls(cls) -> str:
|
||||
return "vllm_omni.platforms.musa.worker.musa_generation_worker.MUSAGenerationWorker"
|
||||
|
||||
@classmethod
|
||||
def get_default_stage_config_path(cls) -> str:
|
||||
return "vllm_omni/model_executor/stage_configs"
|
||||
|
||||
@classmethod
|
||||
def get_diffusion_model_impl_qualname(cls, op_name: str) -> str:
|
||||
# MUSA uses default implementations for diffusion ops
|
||||
if op_name == "hunyuan_fused_moe":
|
||||
return "vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe.HunyuanFusedMoEDefault"
|
||||
return super().get_diffusion_model_impl_qualname(op_name)
|
||||
|
||||
@classmethod
|
||||
def prepare_diffusion_op_runtime(cls, op_name: str, **kwargs: Any) -> None:
|
||||
# MUSA uses default runtime preparation
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def get_diffusion_attn_backend_cls(
|
||||
cls,
|
||||
selected_backend: str | None,
|
||||
head_size: int,
|
||||
) -> str:
|
||||
"""Get the diffusion attention backend class path for MUSA platform.
|
||||
|
||||
MUSA currently supports SDPA (Scaled Dot Product Attention) as the
|
||||
primary backend. Flash Attention support may be added in future
|
||||
when MUSA-specific implementations are available.
|
||||
|
||||
Args:
|
||||
selected_backend: User-selected backend name (e.g., "FLASH_ATTN",
|
||||
"TORCH_SDPA"). If None, uses platform default.
|
||||
head_size: Attention head size.
|
||||
|
||||
Returns:
|
||||
Fully qualified class path of the selected backend.
|
||||
"""
|
||||
if selected_backend is not None:
|
||||
backend_upper = selected_backend.upper()
|
||||
backend = DiffusionAttentionBackendEnum[backend_upper]
|
||||
logger.info("Using diffusion attention backend '%s'", backend_upper)
|
||||
return backend.get_path()
|
||||
|
||||
# Default to SDPA for MUSA as it's the most compatible backend
|
||||
logger.info("Defaulting to diffusion attention backend SDPA")
|
||||
return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path()
|
||||
|
||||
@classmethod
|
||||
def supports_torch_inductor(cls) -> bool:
|
||||
"""MUSA supports torch.compile with inductor backend."""
|
||||
return True
|
||||
|
||||
@classmethod
|
||||
def get_torch_device(cls, local_rank: int | None = None) -> torch.device:
|
||||
"""Get the torch device for MUSA platform.
|
||||
|
||||
Args:
|
||||
local_rank: Optional local rank for multi-GPU setups.
|
||||
|
||||
Returns:
|
||||
torch.device for MUSA GPU.
|
||||
"""
|
||||
if local_rank is None:
|
||||
return torch.device("musa")
|
||||
return torch.device("musa", local_rank)
|
||||
|
||||
@classmethod
|
||||
def get_device_count(cls) -> int:
|
||||
"""Get the number of available MUSA devices."""
|
||||
return torch.musa.device_count()
|
||||
|
||||
@classmethod
|
||||
def synchronize(cls) -> None:
|
||||
"""Synchronize all MUSA operations."""
|
||||
torch.musa.synchronize()
|
||||
|
||||
@classmethod
|
||||
def get_free_memory(cls, device: torch.device | None = None) -> int:
|
||||
"""Get the free memory on the MUSA device.
|
||||
|
||||
Args:
|
||||
device: Optional device to query. If None, uses current device.
|
||||
|
||||
Returns:
|
||||
Free memory in bytes.
|
||||
"""
|
||||
free, _ = torch.musa.mem_get_info(device)
|
||||
return free
|
||||
|
||||
@classmethod
|
||||
def get_device_name(cls, device_id: int = 0) -> str:
|
||||
return torch.musa.get_device_name(device_id)
|
||||
@@ -0,0 +1,9 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from vllm_omni.platforms.musa.worker.musa_ar_worker import MUSAARWorker
|
||||
from vllm_omni.platforms.musa.worker.musa_generation_worker import (
|
||||
MUSAGenerationWorker,
|
||||
)
|
||||
|
||||
__all__ = ["MUSAARWorker", "MUSAGenerationWorker"]
|
||||
@@ -0,0 +1,103 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""MUSA AR (Autoregressive) worker for vLLM-Omni.
|
||||
|
||||
This worker handles autoregressive model stages (thinker/talker) on MUSA devices.
|
||||
"""
|
||||
|
||||
import gc
|
||||
import os
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.utils import report_usage_stats
|
||||
from vllm.v1.worker.gpu_worker import init_worker_distributed_environment
|
||||
from vllm.v1.worker.utils import request_memory
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from vllm_omni.worker.base import OmniGPUWorkerBase
|
||||
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
|
||||
from vllm_omni.worker.mixins import OmniWorkerMixin
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MUSAARWorker(OmniWorkerMixin, OmniGPUWorkerBase):
|
||||
"""MUSA AR worker for thinker/talker stages in Omni model."""
|
||||
|
||||
def init_device(self):
|
||||
"""Initialize the MUSA device for this worker."""
|
||||
# This env var set by Ray causes exceptions with graph building.
|
||||
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
|
||||
parallel_config = self.parallel_config
|
||||
if (
|
||||
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
):
|
||||
# Use local DP rank if available, otherwise use global DP rank.
|
||||
dp_local_rank = self.parallel_config.data_parallel_rank_local
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = self.parallel_config.data_parallel_index
|
||||
|
||||
tp_pp_world_size = self.parallel_config.pipeline_parallel_size * self.parallel_config.tensor_parallel_size
|
||||
|
||||
# DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK
|
||||
self.local_rank += dp_local_rank * tp_pp_world_size
|
||||
assert self.local_rank < torch.musa.device_count(), (
|
||||
f"DP adjusted local rank {self.local_rank} is out of bounds. "
|
||||
)
|
||||
visible_device_count = torch.musa.device_count()
|
||||
assert self.parallel_config.local_world_size <= visible_device_count, (
|
||||
f"local_world_size ({self.parallel_config.local_world_size}) must "
|
||||
f"be less than or equal to the number of visible devices "
|
||||
f"({visible_device_count})."
|
||||
)
|
||||
|
||||
self.device = torch.device(f"musa:{self.local_rank}")
|
||||
torch.musa.set_device(self.device)
|
||||
|
||||
current_platform.check_if_supports_dtype(self.model_config.dtype)
|
||||
|
||||
# Initialize the distributed environment BEFORE taking memory snapshot
|
||||
# This ensures NCCL buffers are allocated before we measure available memory
|
||||
init_worker_distributed_environment(
|
||||
self.vllm_config,
|
||||
self.rank,
|
||||
self.distributed_init_method,
|
||||
self.local_rank,
|
||||
current_platform.dist_backend,
|
||||
)
|
||||
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Now take memory snapshot after distributed environment is initialized
|
||||
gc.collect()
|
||||
torch.musa.empty_cache()
|
||||
|
||||
# Take current memory snapshot
|
||||
self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device)
|
||||
self.requested_memory = request_memory(init_snapshot, self.cache_config)
|
||||
logger.debug("worker init memory snapshot: %r", self.init_snapshot)
|
||||
logger.debug("worker requested memory: %sGiB", format_gib(self.requested_memory))
|
||||
|
||||
# Initialize workspace manager
|
||||
num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1
|
||||
init_workspace_manager(self.device, num_ubatches)
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
# OMNI: v2 model runner does not yet include omni hooks.
|
||||
logger.warning("OMNI MUSAARWorker forces v1 model runner for omni hooks.")
|
||||
self.use_v2_model_runner = False
|
||||
|
||||
# Construct the model runner
|
||||
self.model_runner = GPUARModelRunner(self.vllm_config, self.device)
|
||||
|
||||
if self.rank == 0:
|
||||
# If usage stat is enabled, collect relevant info.
|
||||
report_usage_stats(self.vllm_config)
|
||||
@@ -0,0 +1,106 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""MUSA generation worker for vLLM-Omni.
|
||||
|
||||
This worker handles non-autoregressive generation stages (e.g., code2wav waveform
|
||||
generation) on MUSA devices.
|
||||
"""
|
||||
|
||||
import gc
|
||||
import os
|
||||
|
||||
import torch
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.tracing import instrument
|
||||
from vllm.utils.mem_utils import MemorySnapshot, format_gib
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.v1.utils import report_usage_stats
|
||||
from vllm.v1.worker.gpu_worker import init_worker_distributed_environment
|
||||
from vllm.v1.worker.utils import request_memory
|
||||
from vllm.v1.worker.workspace import init_workspace_manager
|
||||
|
||||
from vllm_omni.worker.base import OmniGPUWorkerBase
|
||||
from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner
|
||||
from vllm_omni.worker.mixins import OmniWorkerMixin
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class MUSAGenerationWorker(OmniWorkerMixin, OmniGPUWorkerBase):
|
||||
"""MUSA generation worker for non-AR waveform generation stage."""
|
||||
|
||||
@instrument(span_name="Init device")
|
||||
def init_device(self):
|
||||
"""Initialize the MUSA device for this worker."""
|
||||
# This env var set by Ray causes exceptions with graph building.
|
||||
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
|
||||
parallel_config = self.parallel_config
|
||||
if (
|
||||
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
|
||||
and parallel_config.data_parallel_backend != "ray"
|
||||
and parallel_config.nnodes_within_dp == 1
|
||||
):
|
||||
# Use local DP rank if available, otherwise use global DP rank.
|
||||
dp_local_rank = self.parallel_config.data_parallel_rank_local
|
||||
if dp_local_rank is None:
|
||||
dp_local_rank = self.parallel_config.data_parallel_index
|
||||
|
||||
tp_pp_world_size = self.parallel_config.pipeline_parallel_size * self.parallel_config.tensor_parallel_size
|
||||
|
||||
# DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK
|
||||
self.local_rank += dp_local_rank * tp_pp_world_size
|
||||
assert self.local_rank < torch.musa.device_count(), (
|
||||
f"DP adjusted local rank {self.local_rank} is out of bounds. "
|
||||
)
|
||||
visible_device_count = torch.musa.device_count()
|
||||
assert self.parallel_config.local_world_size <= visible_device_count, (
|
||||
f"local_world_size ({self.parallel_config.local_world_size}) must "
|
||||
f"be less than or equal to the number of visible devices "
|
||||
f"({visible_device_count})."
|
||||
)
|
||||
|
||||
self.device = torch.device(f"musa:{self.local_rank}")
|
||||
torch.musa.set_device(self.device)
|
||||
|
||||
current_platform.check_if_supports_dtype(self.model_config.dtype)
|
||||
|
||||
# Initialize the distributed environment BEFORE taking memory snapshot
|
||||
# This ensures NCCL buffers are allocated before we measure available memory
|
||||
init_worker_distributed_environment(
|
||||
self.vllm_config,
|
||||
self.rank,
|
||||
self.distributed_init_method,
|
||||
self.local_rank,
|
||||
current_platform.dist_backend,
|
||||
)
|
||||
|
||||
# Set random seed.
|
||||
set_random_seed(self.model_config.seed)
|
||||
|
||||
# Now take memory snapshot after distributed environment is initialized
|
||||
gc.collect()
|
||||
torch.musa.empty_cache()
|
||||
|
||||
# Take current memory snapshot
|
||||
self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device)
|
||||
self.requested_memory = request_memory(init_snapshot, self.cache_config)
|
||||
logger.debug("worker init memory snapshot: %r", self.init_snapshot)
|
||||
logger.debug("worker requested memory: %sGiB", format_gib(self.requested_memory))
|
||||
|
||||
# Initialize workspace manager
|
||||
num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1
|
||||
init_workspace_manager(self.device, num_ubatches)
|
||||
|
||||
if self.use_v2_model_runner:
|
||||
# OMNI: v2 model runner does not yet include omni hooks.
|
||||
logger.warning("OMNI MUSAGenerationWorker forces v1 model runner for omni hooks.")
|
||||
self.use_v2_model_runner = False
|
||||
|
||||
# Construct the model runner
|
||||
self.model_runner = GPUGenerationModelRunner(self.vllm_config, self.device)
|
||||
|
||||
if self.rank == 0:
|
||||
# If usage stat is enabled, collect relevant info.
|
||||
report_usage_stats(self.vllm_config)
|
||||
@@ -18,11 +18,12 @@ from vllm.profiler.wrapper import WorkerProfiler
|
||||
logger = init_logger(__name__)
|
||||
|
||||
# NPU has its custom profiler
|
||||
TorchProfilerActivity = Literal["CPU", "CUDA", "XPU", "NPU"]
|
||||
TorchProfilerActivity = Literal["CPU", "CUDA", "XPU", "NPU", "MUSA"]
|
||||
TorchProfilerActivityMap = {
|
||||
"CPU": torch.profiler.ProfilerActivity.CPU,
|
||||
"CUDA": torch.profiler.ProfilerActivity.CUDA,
|
||||
"XPU": torch.profiler.ProfilerActivity.XPU,
|
||||
"MUSA": torch.profiler.ProfilerActivity.CUDA,
|
||||
}
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user