[Feat] Add MUSA platform support for Moore Threads GPUs (#2337)

Signed-off-by: Xiaodong Ye <yeahdongcn@gmail.com>
This commit is contained in:
R0CKSTAR authored and GitHub committed 2026-04-01 17:55:00 +08:00
1 parent bf5bd0a4c0
commit 3def008b32
15 files changed
+454 -11

No files matched your search

+4
View File
@@ -177,20 +177,24 @@ markers = [
"rocm: Tests that run on AMD/ROCm (auto-added)",
"xpu: Tests that run on XPU (auto-added)",
"npu: Tests that run on NPU/Ascend (auto-added)",
"musa: Tests that run on MUSA/Moore Threads (auto-added)",
# specified computation resources marks (auto-added)
"H100: Tests that require H100 GPU",
"L4: Tests that require L4 GPU",
"MI325: Tests that require MI325 GPU (AMD/ROCm)",
"S5000: Tests that require S5000 GPU (Moore Threads/MUSA)",
"A2: Tests that require A2 NPU",
"A3: Tests that require A3 NPU",
"distributed_cuda: Tests that require multi cards on CUDA platform",
"distributed_rocm: Tests that require multi cards on ROCm platform",
"distributed_xpu: Tests that require multi cards on XPU platform",
"distributed_npu: Tests that require multi cards on NPU platform",
"distributed_musa: Tests that require multi cards on MUSA platform",
"skipif_cuda: Skip if the num of CUDA cards is less than the required",
"skipif_rocm: Skip if the num of ROCm cards is less than the required",
"skipif_xpu: Skip if the num of XPU cards is less than the required",
"skipif_npu: Skip if the num of NPU cards is less than the required",
"skipif_musa: Skip if the num of MUSA cards is less than the required",
# more detailed markers
"slow: Slow tests (may skip in quick CI)",
"benchmark: Benchmark tests",
+4
View File
@@ -0,0 +1,4 @@
-r common.txt
# MUSA platform dependencies
torchada>=0.1.46
onnxruntime>=1.23.2
+15 -4
View File
@@ -3,7 +3,7 @@ Setup script for vLLM-Omni with hardware-dependent installation.
This setup.py implements platform-aware dependency routing so users can run
`pip install vllm-omni` and automatically receive the correct platform-specific
dependencies (CUDA/ROCm/CPU/XPU/NPU) without requiring extras like `[cuda]`.
dependencies (CUDA/ROCm/CPU/XPU/NPU/MUSA) without requiring extras like `[cuda]`.
"""
import os
@@ -46,16 +46,16 @@ def detect_target_device() -> str:
Priority order:
1. VLLM_OMNI_TARGET_DEVICE environment variable (highest priority)
2. Torch backend detection (cuda, rocm, npu, xpu)
2. Torch backend detection (cuda, rocm, npu, xpu, musa)
3. CPU fallback (default)
Returns:
str: Device name ('cuda', 'rocm', 'npu', 'xpu', or 'cpu')
str: Device name ('cuda', 'rocm', 'npu', 'xpu', 'musa', or 'cpu')
"""
# Priority 1: Explicit override via environment variable
target_device = os.environ.get("VLLM_OMNI_TARGET_DEVICE")
if target_device:
valid_devices = ["cuda", "rocm", "npu", "xpu", "cpu"]
valid_devices = ["cuda", "rocm", "npu", "xpu", "musa", "cpu"]
if target_device.lower() in valid_devices:
print(f"Using target device from VLLM_OMNI_TARGET_DEVICE: {target_device.lower()}")
return target_device.lower()
@@ -97,6 +97,15 @@ def detect_target_device() -> str:
except Exception:
pass
# Check for MUSA (Moore Threads)
if hasattr(torch, "musa"):
try:
if torch.musa.is_available():
print("Detected MUSA backend from torch")
return "musa"
except Exception:
pass
print("No GPU backend detected in torch, defaulting to CPU")
return "cpu"
@@ -152,6 +161,8 @@ def get_vllm_omni_version() -> str:
version += f"{sep}npu"
elif device == "xpu":
version += f"{sep}xpu"
elif device == "musa":
version += f"{sep}musa"
elif device == "cpu":
version += f"{sep}cpu"
else:
+40 -6
View File
@@ -430,13 +430,41 @@ def xpu_marks(*, res: str, num_cards: int):
return marks + [test_distributed]
def musa_marks(*, res: str, num_cards: int):
"""
Get a collection of pytest marks to apply for `@musa_test`.
Args:
res: Resource type, e.g., "S5000".
num_cards: Number of GPU cards required.
Returns:
List of pytest marks to apply.
"""
test_platform_detail = pytest.mark.musa
if res == "S5000":
test_resource = pytest.mark.S5000
else:
raise ValueError(f"Invalid MUSA resource type: {res}. Supported: S5000")
marks = [test_resource, test_platform_detail]
if num_cards == 1:
return marks
else:
test_distributed = pytest.mark.distributed_musa(num_cards=num_cards)
# TODO: add MUSA support for `skipif_musa` marker
return marks + [test_distributed]
def gpu_marks(*, res: str, num_cards: int):
"""
Get a collection of pytest marks to apply for `@gpu_test`.
Platform is automatically determined based on resource type.
Args:
res: Resource type, e.g., "L4", "H100" for CUDA, or "MI325" for ROCm, or "B60" for XPU.
res: Resource type, e.g., "L4", "H100" for CUDA, or "MI325" for ROCm, or "B60" for XPU, or "S5000" for MUSA.
num_cards: Number of GPU cards required.
Returns:
@@ -449,7 +477,9 @@ def gpu_marks(*, res: str, num_cards: int):
return [test_platform] + rocm_marks(res=res, num_cards=num_cards)
if res == "B60":
return [test_platform] + xpu_marks(res=res, num_cards=num_cards)
raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325")
if res == "S5000":
return [test_platform] + musa_marks(res=res, num_cards=num_cards)
raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325, B60, S5000")
def npu_marks(*, res: str, num_cards: int):
@@ -476,13 +506,13 @@ def npu_marks(*, res: str, num_cards: int):
def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
"""
Get a collection of pytest marks to apply for `@hardware_test`,
including CUDA, ROCm, XPU, and NPU,
including CUDA, ROCm, XPU, NPU, and MUSA,
based on the specified platforms and resources.
"""
# Validate platforms
# Don't validate platform details in this decorator
for platform, _ in res.items():
if platform not in ("cuda", "rocm", "xpu", "npu"):
if platform not in ("cuda", "rocm", "xpu", "npu", "musa"):
raise ValueError(f"Unsupported platform: {platform}")
# Normalize num_cards
@@ -505,6 +535,8 @@ def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
cards = num_cards_dict[platform]
if platform == "cuda" or platform == "rocm" or platform == "xpu":
marks = gpu_marks(res=resource, num_cards=cards)
elif platform == "musa":
marks = musa_marks(res=resource, num_cards=cards)
elif platform == "npu":
marks = npu_marks(res=resource, num_cards=cards)
else:
@@ -522,15 +554,17 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1):
res: Mapping from platform to resource type. Supported platforms/resources:
- cuda: L4, H100
- rocm: MI325
- xpu: B60
- npu: A2, A3
- musa: S5000
num_cards: Number of cards required. Can be:
- int: same card count for all platforms (default: 1)
- dict: per-platform card count, e.g., {"cuda": 2, "rocm": 2}
Example:
@hardware_test(
res={"cuda": "L4", "rocm": "MI325", "npu": "A2"},
num_cards={"cuda": 2, "rocm": 2, "npu": 2},
res={"cuda": "L4", "rocm": "MI325", "npu": "A2", "musa": "S5000"},
num_cards={"cuda": 2, "rocm": 2, "npu": 2, "musa": 2},
)
def test_multi_platform():
...
@@ -99,6 +99,8 @@ class AttentionImpl(ABC, Generic[T]):
return self.forward_npu(query, key, value, attn_metadata)
elif current_omni_platform.is_xpu():
return self.forward_xpu(query, key, value, attn_metadata)
elif current_omni_platform.is_musa():
return self.forward_musa(query, key, value, attn_metadata)
else:
raise NotImplementedError(f"No forward implementation for platform: {current_omni_platform}")
@@ -138,3 +140,13 @@ class AttentionImpl(ABC, Generic[T]):
) -> torch.Tensor:
# By default, HIP ops are compatible with CUDA ops.
return self.forward_cuda(query, key, value, attn_metadata)
def forward_musa(
self,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attn_metadata: T | None = None,
) -> torch.Tensor:
# By default, MUSA ops are compatible with CUDA ops.
return self.forward_cuda(query, key, value, attn_metadata)
@@ -35,6 +35,9 @@ elif current_omni_platform.is_xpu():
from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func # noqa: F401
except (ImportError, ModuleNotFoundError):
pass
elif current_omni_platform.is_musa():
# XXX (MUSA): Add MUSA-specific Flash Attention when available
pass
else:
# CUDA: try FA3 -> FA2 fallback chain
# Try FA3 from fa3-fwd PyPI package
+6
View File
@@ -25,6 +25,8 @@ class CustomOp(nn.Module):
return self.forward_npu
elif current_omni_platform.is_xpu():
return self.forward_xpu
elif current_omni_platform.is_musa():
return self.forward_musa
else:
return self.forward_native
@@ -51,3 +53,7 @@ class CustomOp(nn.Module):
def forward_hip(self, *args, **kwargs):
# By default, we assume that HIP ops are compatible with CUDA ops.
return self.forward_cuda(*args, **kwargs)
def forward_musa(self, *args, **kwargs):
# By default, we assume that MUSA ops are compatible with CUDA ops.
return self.forward_cuda(*args, **kwargs)
+17
View File
@@ -105,11 +105,28 @@ def xpu_omni_platform_plugin() -> str | None:
return "vllm_omni.platforms.xpu.platform.XPUOmniPlatform" if is_xpu else None
def musa_omni_platform_plugin() -> str | None:
"""Check if MUSA OmniPlatform should be activated."""
is_musa = False
logger.debug("Checking if MUSA OmniPlatform is available.")
try:
import torchada
if torchada.is_musa_platform():
is_musa = True
logger.debug("Confirmed MUSA OmniPlatform is available.")
except Exception as e:
logger.debug("MUSA OmniPlatform is not available because: %s", str(e))
return "vllm_omni.platforms.musa.platform.MUSAOmniPlatform" if is_musa else None
builtin_omni_platform_plugins = {
"cuda": cuda_omni_platform_plugin,
"rocm": rocm_omni_platform_plugin,
"npu": npu_omni_platform_plugin,
"xpu": xpu_omni_platform_plugin,
"musa": musa_omni_platform_plugin,
}
+4
View File
@@ -15,6 +15,7 @@ class OmniPlatformEnum(Enum):
ROCM = "rocm"
NPU = "npu"
XPU = "xpu"
MUSA = "musa"
UNSPECIFIED = "unspecified"
@@ -41,6 +42,9 @@ class OmniPlatform(Platform):
def is_rocm(self) -> bool:
return self._omni_enum == OmniPlatformEnum.ROCM
def is_musa(self) -> bool:
return self._omni_enum == OmniPlatformEnum.MUSA
@classmethod
def get_omni_ar_worker_cls(cls) -> str:
raise NotImplementedError
+6
View File
@@ -0,0 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm_omni.platforms.musa.platform import MUSAOmniPlatform
__all__ = ["MUSAOmniPlatform"]
+123
View File
@@ -0,0 +1,123 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import torch
from vllm.logger import init_logger
from vllm_musa.platform import MUSAPlatformBase
from vllm_omni.diffusion.attention.backends.registry import DiffusionAttentionBackendEnum
from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum
logger = init_logger(__name__)
class MUSAOmniPlatform(OmniPlatform, MUSAPlatformBase):
"""MUSA/Moore Threads GPU implementation of OmniPlatform.
Inherits all MUSA-specific implementations from vllm-musa's MUSAPlatformBase,
and adds Omni-specific interfaces from OmniPlatform.
"""
_omni_enum = OmniPlatformEnum.MUSA
@classmethod
def get_omni_ar_worker_cls(cls) -> str:
return "vllm_omni.platforms.musa.worker.musa_ar_worker.MUSAARWorker"
@classmethod
def get_omni_generation_worker_cls(cls) -> str:
return "vllm_omni.platforms.musa.worker.musa_generation_worker.MUSAGenerationWorker"
@classmethod
def get_default_stage_config_path(cls) -> str:
return "vllm_omni/model_executor/stage_configs"
@classmethod
def get_diffusion_model_impl_qualname(cls, op_name: str) -> str:
# MUSA uses default implementations for diffusion ops
if op_name == "hunyuan_fused_moe":
return "vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe.HunyuanFusedMoEDefault"
return super().get_diffusion_model_impl_qualname(op_name)
@classmethod
def prepare_diffusion_op_runtime(cls, op_name: str, **kwargs: Any) -> None:
# MUSA uses default runtime preparation
return None
@classmethod
def get_diffusion_attn_backend_cls(
cls,
selected_backend: str | None,
head_size: int,
) -> str:
"""Get the diffusion attention backend class path for MUSA platform.
MUSA currently supports SDPA (Scaled Dot Product Attention) as the
primary backend. Flash Attention support may be added in future
when MUSA-specific implementations are available.
Args:
selected_backend: User-selected backend name (e.g., "FLASH_ATTN",
"TORCH_SDPA"). If None, uses platform default.
head_size: Attention head size.
Returns:
Fully qualified class path of the selected backend.
"""
if selected_backend is not None:
backend_upper = selected_backend.upper()
backend = DiffusionAttentionBackendEnum[backend_upper]
logger.info("Using diffusion attention backend '%s'", backend_upper)
return backend.get_path()
# Default to SDPA for MUSA as it's the most compatible backend
logger.info("Defaulting to diffusion attention backend SDPA")
return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path()
@classmethod
def supports_torch_inductor(cls) -> bool:
"""MUSA supports torch.compile with inductor backend."""
return True
@classmethod
def get_torch_device(cls, local_rank: int | None = None) -> torch.device:
"""Get the torch device for MUSA platform.
Args:
local_rank: Optional local rank for multi-GPU setups.
Returns:
torch.device for MUSA GPU.
"""
if local_rank is None:
return torch.device("musa")
return torch.device("musa", local_rank)
@classmethod
def get_device_count(cls) -> int:
"""Get the number of available MUSA devices."""
return torch.musa.device_count()
@classmethod
def synchronize(cls) -> None:
"""Synchronize all MUSA operations."""
torch.musa.synchronize()
@classmethod
def get_free_memory(cls, device: torch.device | None = None) -> int:
"""Get the free memory on the MUSA device.
Args:
device: Optional device to query. If None, uses current device.
Returns:
Free memory in bytes.
"""
free, _ = torch.musa.mem_get_info(device)
return free
@classmethod
def get_device_name(cls, device_id: int = 0) -> str:
return torch.musa.get_device_name(device_id)
@@ -0,0 +1,9 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm_omni.platforms.musa.worker.musa_ar_worker import MUSAARWorker
from vllm_omni.platforms.musa.worker.musa_generation_worker import (
MUSAGenerationWorker,
)
__all__ = ["MUSAARWorker", "MUSAGenerationWorker"]
@@ -0,0 +1,103 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MUSA AR (Autoregressive) worker for vLLM-Omni.
This worker handles autoregressive model stages (thinker/talker) on MUSA devices.
"""
import gc
import os
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.mem_utils import MemorySnapshot, format_gib
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.utils import report_usage_stats
from vllm.v1.worker.gpu_worker import init_worker_distributed_environment
from vllm.v1.worker.utils import request_memory
from vllm.v1.worker.workspace import init_workspace_manager
from vllm_omni.worker.base import OmniGPUWorkerBase
from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner
from vllm_omni.worker.mixins import OmniWorkerMixin
logger = init_logger(__name__)
class MUSAARWorker(OmniWorkerMixin, OmniGPUWorkerBase):
"""MUSA AR worker for thinker/talker stages in Omni model."""
def init_device(self):
"""Initialize the MUSA device for this worker."""
# This env var set by Ray causes exceptions with graph building.
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
parallel_config = self.parallel_config
if (
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
and parallel_config.data_parallel_backend != "ray"
and parallel_config.nnodes_within_dp == 1
):
# Use local DP rank if available, otherwise use global DP rank.
dp_local_rank = self.parallel_config.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = self.parallel_config.data_parallel_index
tp_pp_world_size = self.parallel_config.pipeline_parallel_size * self.parallel_config.tensor_parallel_size
# DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK
self.local_rank += dp_local_rank * tp_pp_world_size
assert self.local_rank < torch.musa.device_count(), (
f"DP adjusted local rank {self.local_rank} is out of bounds. "
)
visible_device_count = torch.musa.device_count()
assert self.parallel_config.local_world_size <= visible_device_count, (
f"local_world_size ({self.parallel_config.local_world_size}) must "
f"be less than or equal to the number of visible devices "
f"({visible_device_count})."
)
self.device = torch.device(f"musa:{self.local_rank}")
torch.musa.set_device(self.device)
current_platform.check_if_supports_dtype(self.model_config.dtype)
# Initialize the distributed environment BEFORE taking memory snapshot
# This ensures NCCL buffers are allocated before we measure available memory
init_worker_distributed_environment(
self.vllm_config,
self.rank,
self.distributed_init_method,
self.local_rank,
current_platform.dist_backend,
)
# Set random seed.
set_random_seed(self.model_config.seed)
# Now take memory snapshot after distributed environment is initialized
gc.collect()
torch.musa.empty_cache()
# Take current memory snapshot
self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device)
self.requested_memory = request_memory(init_snapshot, self.cache_config)
logger.debug("worker init memory snapshot: %r", self.init_snapshot)
logger.debug("worker requested memory: %sGiB", format_gib(self.requested_memory))
# Initialize workspace manager
num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1
init_workspace_manager(self.device, num_ubatches)
if self.use_v2_model_runner:
# OMNI: v2 model runner does not yet include omni hooks.
logger.warning("OMNI MUSAARWorker forces v1 model runner for omni hooks.")
self.use_v2_model_runner = False
# Construct the model runner
self.model_runner = GPUARModelRunner(self.vllm_config, self.device)
if self.rank == 0:
# If usage stat is enabled, collect relevant info.
report_usage_stats(self.vllm_config)
@@ -0,0 +1,106 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MUSA generation worker for vLLM-Omni.
This worker handles non-autoregressive generation stages (e.g., code2wav waveform
generation) on MUSA devices.
"""
import gc
import os
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.tracing import instrument
from vllm.utils.mem_utils import MemorySnapshot, format_gib
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.utils import report_usage_stats
from vllm.v1.worker.gpu_worker import init_worker_distributed_environment
from vllm.v1.worker.utils import request_memory
from vllm.v1.worker.workspace import init_workspace_manager
from vllm_omni.worker.base import OmniGPUWorkerBase
from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner
from vllm_omni.worker.mixins import OmniWorkerMixin
logger = init_logger(__name__)
class MUSAGenerationWorker(OmniWorkerMixin, OmniGPUWorkerBase):
"""MUSA generation worker for non-AR waveform generation stage."""
@instrument(span_name="Init device")
def init_device(self):
"""Initialize the MUSA device for this worker."""
# This env var set by Ray causes exceptions with graph building.
os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None)
parallel_config = self.parallel_config
if (
parallel_config.distributed_executor_backend not in ("ray", "external_launcher")
and parallel_config.data_parallel_backend != "ray"
and parallel_config.nnodes_within_dp == 1
):
# Use local DP rank if available, otherwise use global DP rank.
dp_local_rank = self.parallel_config.data_parallel_rank_local
if dp_local_rank is None:
dp_local_rank = self.parallel_config.data_parallel_index
tp_pp_world_size = self.parallel_config.pipeline_parallel_size * self.parallel_config.tensor_parallel_size
# DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK
self.local_rank += dp_local_rank * tp_pp_world_size
assert self.local_rank < torch.musa.device_count(), (
f"DP adjusted local rank {self.local_rank} is out of bounds. "
)
visible_device_count = torch.musa.device_count()
assert self.parallel_config.local_world_size <= visible_device_count, (
f"local_world_size ({self.parallel_config.local_world_size}) must "
f"be less than or equal to the number of visible devices "
f"({visible_device_count})."
)
self.device = torch.device(f"musa:{self.local_rank}")
torch.musa.set_device(self.device)
current_platform.check_if_supports_dtype(self.model_config.dtype)
# Initialize the distributed environment BEFORE taking memory snapshot
# This ensures NCCL buffers are allocated before we measure available memory
init_worker_distributed_environment(
self.vllm_config,
self.rank,
self.distributed_init_method,
self.local_rank,
current_platform.dist_backend,
)
# Set random seed.
set_random_seed(self.model_config.seed)
# Now take memory snapshot after distributed environment is initialized
gc.collect()
torch.musa.empty_cache()
# Take current memory snapshot
self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device)
self.requested_memory = request_memory(init_snapshot, self.cache_config)
logger.debug("worker init memory snapshot: %r", self.init_snapshot)
logger.debug("worker requested memory: %sGiB", format_gib(self.requested_memory))
# Initialize workspace manager
num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1
init_workspace_manager(self.device, num_ubatches)
if self.use_v2_model_runner:
# OMNI: v2 model runner does not yet include omni hooks.
logger.warning("OMNI MUSAGenerationWorker forces v1 model runner for omni hooks.")
self.use_v2_model_runner = False
# Construct the model runner
self.model_runner = GPUGenerationModelRunner(self.vllm_config, self.device)
if self.rank == 0:
# If usage stat is enabled, collect relevant info.
report_usage_stats(self.vllm_config)
+2 -1
View File
@@ -18,11 +18,12 @@ from vllm.profiler.wrapper import WorkerProfiler
logger = init_logger(__name__)
# NPU has its custom profiler
TorchProfilerActivity = Literal["CPU", "CUDA", "XPU", "NPU"]
TorchProfilerActivity = Literal["CPU", "CUDA", "XPU", "NPU", "MUSA"]
TorchProfilerActivityMap = {
"CPU": torch.profiler.ProfilerActivity.CPU,
"CUDA": torch.profiler.ProfilerActivity.CUDA,
"XPU": torch.profiler.ProfilerActivity.XPU,
"MUSA": torch.profiler.ProfilerActivity.CUDA,
}