From 3def008b324f636940953862daedec54b9021a87 Mon Sep 17 00:00:00 2001 From: R0CKSTAR Date: Wed, 1 Apr 2026 17:55:00 +0800 Subject: [PATCH] [Feat] Add MUSA platform support for Moore Threads GPUs (#2337) Signed-off-by: Xiaodong Ye --- pyproject.toml | 4 + requirements/musa.txt | 4 + setup.py | 19 ++- tests/utils.py | 46 ++++++- .../diffusion/attention/backends/abstract.py | 12 ++ .../diffusion/attention/backends/utils/fa.py | 3 + vllm_omni/diffusion/layers/custom_op.py | 6 + vllm_omni/platforms/__init__.py | 17 +++ vllm_omni/platforms/interface.py | 4 + vllm_omni/platforms/musa/__init__.py | 6 + vllm_omni/platforms/musa/platform.py | 123 ++++++++++++++++++ vllm_omni/platforms/musa/worker/__init__.py | 9 ++ .../platforms/musa/worker/musa_ar_worker.py | 103 +++++++++++++++ .../musa/worker/musa_generation_worker.py | 106 +++++++++++++++ vllm_omni/profiler/omni_torch_profiler.py | 3 +- 15 files changed, 454 insertions(+), 11 deletions(-) create mode 100644 requirements/musa.txt create mode 100644 vllm_omni/platforms/musa/__init__.py create mode 100644 vllm_omni/platforms/musa/platform.py create mode 100644 vllm_omni/platforms/musa/worker/__init__.py create mode 100644 vllm_omni/platforms/musa/worker/musa_ar_worker.py create mode 100644 vllm_omni/platforms/musa/worker/musa_generation_worker.py diff --git a/pyproject.toml b/pyproject.toml index 43e9506fd..15e7c6305 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,20 +177,24 @@ markers = [ "rocm: Tests that run on AMD/ROCm (auto-added)", "xpu: Tests that run on XPU (auto-added)", "npu: Tests that run on NPU/Ascend (auto-added)", + "musa: Tests that run on MUSA/Moore Threads (auto-added)", # specified computation resources marks (auto-added) "H100: Tests that require H100 GPU", "L4: Tests that require L4 GPU", "MI325: Tests that require MI325 GPU (AMD/ROCm)", + "S5000: Tests that require S5000 GPU (Moore Threads/MUSA)", "A2: Tests that require A2 NPU", "A3: Tests that require A3 NPU", "distributed_cuda: Tests that require multi cards on CUDA platform", "distributed_rocm: Tests that require multi cards on ROCm platform", "distributed_xpu: Tests that require multi cards on XPU platform", "distributed_npu: Tests that require multi cards on NPU platform", + "distributed_musa: Tests that require multi cards on MUSA platform", "skipif_cuda: Skip if the num of CUDA cards is less than the required", "skipif_rocm: Skip if the num of ROCm cards is less than the required", "skipif_xpu: Skip if the num of XPU cards is less than the required", "skipif_npu: Skip if the num of NPU cards is less than the required", + "skipif_musa: Skip if the num of MUSA cards is less than the required", # more detailed markers "slow: Slow tests (may skip in quick CI)", "benchmark: Benchmark tests", diff --git a/requirements/musa.txt b/requirements/musa.txt new file mode 100644 index 000000000..112f32604 --- /dev/null +++ b/requirements/musa.txt @@ -0,0 +1,4 @@ +-r common.txt +# MUSA platform dependencies +torchada>=0.1.46 +onnxruntime>=1.23.2 diff --git a/setup.py b/setup.py index 4ff4936b4..057212d67 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ Setup script for vLLM-Omni with hardware-dependent installation. This setup.py implements platform-aware dependency routing so users can run `pip install vllm-omni` and automatically receive the correct platform-specific -dependencies (CUDA/ROCm/CPU/XPU/NPU) without requiring extras like `[cuda]`. +dependencies (CUDA/ROCm/CPU/XPU/NPU/MUSA) without requiring extras like `[cuda]`. """ import os @@ -46,16 +46,16 @@ def detect_target_device() -> str: Priority order: 1. VLLM_OMNI_TARGET_DEVICE environment variable (highest priority) - 2. Torch backend detection (cuda, rocm, npu, xpu) + 2. Torch backend detection (cuda, rocm, npu, xpu, musa) 3. CPU fallback (default) Returns: - str: Device name ('cuda', 'rocm', 'npu', 'xpu', or 'cpu') + str: Device name ('cuda', 'rocm', 'npu', 'xpu', 'musa', or 'cpu') """ # Priority 1: Explicit override via environment variable target_device = os.environ.get("VLLM_OMNI_TARGET_DEVICE") if target_device: - valid_devices = ["cuda", "rocm", "npu", "xpu", "cpu"] + valid_devices = ["cuda", "rocm", "npu", "xpu", "musa", "cpu"] if target_device.lower() in valid_devices: print(f"Using target device from VLLM_OMNI_TARGET_DEVICE: {target_device.lower()}") return target_device.lower() @@ -97,6 +97,15 @@ def detect_target_device() -> str: except Exception: pass + # Check for MUSA (Moore Threads) + if hasattr(torch, "musa"): + try: + if torch.musa.is_available(): + print("Detected MUSA backend from torch") + return "musa" + except Exception: + pass + print("No GPU backend detected in torch, defaulting to CPU") return "cpu" @@ -152,6 +161,8 @@ def get_vllm_omni_version() -> str: version += f"{sep}npu" elif device == "xpu": version += f"{sep}xpu" + elif device == "musa": + version += f"{sep}musa" elif device == "cpu": version += f"{sep}cpu" else: diff --git a/tests/utils.py b/tests/utils.py index 72fc6639a..84edbbf3d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -430,13 +430,41 @@ def xpu_marks(*, res: str, num_cards: int): return marks + [test_distributed] +def musa_marks(*, res: str, num_cards: int): + """ + Get a collection of pytest marks to apply for `@musa_test`. + + Args: + res: Resource type, e.g., "S5000". + num_cards: Number of GPU cards required. + + Returns: + List of pytest marks to apply. + """ + test_platform_detail = pytest.mark.musa + + if res == "S5000": + test_resource = pytest.mark.S5000 + else: + raise ValueError(f"Invalid MUSA resource type: {res}. Supported: S5000") + + marks = [test_resource, test_platform_detail] + + if num_cards == 1: + return marks + else: + test_distributed = pytest.mark.distributed_musa(num_cards=num_cards) + # TODO: add MUSA support for `skipif_musa` marker + return marks + [test_distributed] + + def gpu_marks(*, res: str, num_cards: int): """ Get a collection of pytest marks to apply for `@gpu_test`. Platform is automatically determined based on resource type. Args: - res: Resource type, e.g., "L4", "H100" for CUDA, or "MI325" for ROCm, or "B60" for XPU. + res: Resource type, e.g., "L4", "H100" for CUDA, or "MI325" for ROCm, or "B60" for XPU, or "S5000" for MUSA. num_cards: Number of GPU cards required. Returns: @@ -449,7 +477,9 @@ def gpu_marks(*, res: str, num_cards: int): return [test_platform] + rocm_marks(res=res, num_cards=num_cards) if res == "B60": return [test_platform] + xpu_marks(res=res, num_cards=num_cards) - raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325") + if res == "S5000": + return [test_platform] + musa_marks(res=res, num_cards=num_cards) + raise ValueError(f"Invalid resource type: {res}. Supported: L4, H100, MI325, B60, S5000") def npu_marks(*, res: str, num_cards: int): @@ -476,13 +506,13 @@ def npu_marks(*, res: str, num_cards: int): def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1): """ Get a collection of pytest marks to apply for `@hardware_test`, - including CUDA, ROCm, XPU, and NPU, + including CUDA, ROCm, XPU, NPU, and MUSA, based on the specified platforms and resources. """ # Validate platforms # Don't validate platform details in this decorator for platform, _ in res.items(): - if platform not in ("cuda", "rocm", "xpu", "npu"): + if platform not in ("cuda", "rocm", "xpu", "npu", "musa"): raise ValueError(f"Unsupported platform: {platform}") # Normalize num_cards @@ -505,6 +535,8 @@ def hardware_marks(*, res: dict[str, str], num_cards: int | dict[str, int] = 1): cards = num_cards_dict[platform] if platform == "cuda" or platform == "rocm" or platform == "xpu": marks = gpu_marks(res=resource, num_cards=cards) + elif platform == "musa": + marks = musa_marks(res=resource, num_cards=cards) elif platform == "npu": marks = npu_marks(res=resource, num_cards=cards) else: @@ -522,15 +554,17 @@ def hardware_test(*, res: dict[str, str], num_cards: int | dict[str, int] = 1): res: Mapping from platform to resource type. Supported platforms/resources: - cuda: L4, H100 - rocm: MI325 + - xpu: B60 - npu: A2, A3 + - musa: S5000 num_cards: Number of cards required. Can be: - int: same card count for all platforms (default: 1) - dict: per-platform card count, e.g., {"cuda": 2, "rocm": 2} Example: @hardware_test( - res={"cuda": "L4", "rocm": "MI325", "npu": "A2"}, - num_cards={"cuda": 2, "rocm": 2, "npu": 2}, + res={"cuda": "L4", "rocm": "MI325", "npu": "A2", "musa": "S5000"}, + num_cards={"cuda": 2, "rocm": 2, "npu": 2, "musa": 2}, ) def test_multi_platform(): ... diff --git a/vllm_omni/diffusion/attention/backends/abstract.py b/vllm_omni/diffusion/attention/backends/abstract.py index d0a62bcd9..472fde422 100644 --- a/vllm_omni/diffusion/attention/backends/abstract.py +++ b/vllm_omni/diffusion/attention/backends/abstract.py @@ -99,6 +99,8 @@ class AttentionImpl(ABC, Generic[T]): return self.forward_npu(query, key, value, attn_metadata) elif current_omni_platform.is_xpu(): return self.forward_xpu(query, key, value, attn_metadata) + elif current_omni_platform.is_musa(): + return self.forward_musa(query, key, value, attn_metadata) else: raise NotImplementedError(f"No forward implementation for platform: {current_omni_platform}") @@ -138,3 +140,13 @@ class AttentionImpl(ABC, Generic[T]): ) -> torch.Tensor: # By default, HIP ops are compatible with CUDA ops. return self.forward_cuda(query, key, value, attn_metadata) + + def forward_musa( + self, + query: torch.Tensor, + key: torch.Tensor, + value: torch.Tensor, + attn_metadata: T | None = None, + ) -> torch.Tensor: + # By default, MUSA ops are compatible with CUDA ops. + return self.forward_cuda(query, key, value, attn_metadata) diff --git a/vllm_omni/diffusion/attention/backends/utils/fa.py b/vllm_omni/diffusion/attention/backends/utils/fa.py index 1474598d7..1fd47790f 100644 --- a/vllm_omni/diffusion/attention/backends/utils/fa.py +++ b/vllm_omni/diffusion/attention/backends/utils/fa.py @@ -35,6 +35,9 @@ elif current_omni_platform.is_xpu(): from vllm.v1.attention.backends.fa_utils import flash_attn_varlen_func # noqa: F401 except (ImportError, ModuleNotFoundError): pass +elif current_omni_platform.is_musa(): + # XXX (MUSA): Add MUSA-specific Flash Attention when available + pass else: # CUDA: try FA3 -> FA2 fallback chain # Try FA3 from fa3-fwd PyPI package diff --git a/vllm_omni/diffusion/layers/custom_op.py b/vllm_omni/diffusion/layers/custom_op.py index 321bcbf8a..27e3bce1f 100644 --- a/vllm_omni/diffusion/layers/custom_op.py +++ b/vllm_omni/diffusion/layers/custom_op.py @@ -25,6 +25,8 @@ class CustomOp(nn.Module): return self.forward_npu elif current_omni_platform.is_xpu(): return self.forward_xpu + elif current_omni_platform.is_musa(): + return self.forward_musa else: return self.forward_native @@ -51,3 +53,7 @@ class CustomOp(nn.Module): def forward_hip(self, *args, **kwargs): # By default, we assume that HIP ops are compatible with CUDA ops. return self.forward_cuda(*args, **kwargs) + + def forward_musa(self, *args, **kwargs): + # By default, we assume that MUSA ops are compatible with CUDA ops. + return self.forward_cuda(*args, **kwargs) diff --git a/vllm_omni/platforms/__init__.py b/vllm_omni/platforms/__init__.py index ae29b71ed..64a7cdb16 100644 --- a/vllm_omni/platforms/__init__.py +++ b/vllm_omni/platforms/__init__.py @@ -105,11 +105,28 @@ def xpu_omni_platform_plugin() -> str | None: return "vllm_omni.platforms.xpu.platform.XPUOmniPlatform" if is_xpu else None +def musa_omni_platform_plugin() -> str | None: + """Check if MUSA OmniPlatform should be activated.""" + is_musa = False + logger.debug("Checking if MUSA OmniPlatform is available.") + try: + import torchada + + if torchada.is_musa_platform(): + is_musa = True + logger.debug("Confirmed MUSA OmniPlatform is available.") + except Exception as e: + logger.debug("MUSA OmniPlatform is not available because: %s", str(e)) + + return "vllm_omni.platforms.musa.platform.MUSAOmniPlatform" if is_musa else None + + builtin_omni_platform_plugins = { "cuda": cuda_omni_platform_plugin, "rocm": rocm_omni_platform_plugin, "npu": npu_omni_platform_plugin, "xpu": xpu_omni_platform_plugin, + "musa": musa_omni_platform_plugin, } diff --git a/vllm_omni/platforms/interface.py b/vllm_omni/platforms/interface.py index 7739cec78..4325851e5 100644 --- a/vllm_omni/platforms/interface.py +++ b/vllm_omni/platforms/interface.py @@ -15,6 +15,7 @@ class OmniPlatformEnum(Enum): ROCM = "rocm" NPU = "npu" XPU = "xpu" + MUSA = "musa" UNSPECIFIED = "unspecified" @@ -41,6 +42,9 @@ class OmniPlatform(Platform): def is_rocm(self) -> bool: return self._omni_enum == OmniPlatformEnum.ROCM + def is_musa(self) -> bool: + return self._omni_enum == OmniPlatformEnum.MUSA + @classmethod def get_omni_ar_worker_cls(cls) -> str: raise NotImplementedError diff --git a/vllm_omni/platforms/musa/__init__.py b/vllm_omni/platforms/musa/__init__.py new file mode 100644 index 000000000..70ea7a962 --- /dev/null +++ b/vllm_omni/platforms/musa/__init__.py @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.platforms.musa.platform import MUSAOmniPlatform + +__all__ = ["MUSAOmniPlatform"] diff --git a/vllm_omni/platforms/musa/platform.py b/vllm_omni/platforms/musa/platform.py new file mode 100644 index 000000000..932ce62d2 --- /dev/null +++ b/vllm_omni/platforms/musa/platform.py @@ -0,0 +1,123 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from typing import Any + +import torch +from vllm.logger import init_logger +from vllm_musa.platform import MUSAPlatformBase + +from vllm_omni.diffusion.attention.backends.registry import DiffusionAttentionBackendEnum +from vllm_omni.platforms.interface import OmniPlatform, OmniPlatformEnum + +logger = init_logger(__name__) + + +class MUSAOmniPlatform(OmniPlatform, MUSAPlatformBase): + """MUSA/Moore Threads GPU implementation of OmniPlatform. + + Inherits all MUSA-specific implementations from vllm-musa's MUSAPlatformBase, + and adds Omni-specific interfaces from OmniPlatform. + """ + + _omni_enum = OmniPlatformEnum.MUSA + + @classmethod + def get_omni_ar_worker_cls(cls) -> str: + return "vllm_omni.platforms.musa.worker.musa_ar_worker.MUSAARWorker" + + @classmethod + def get_omni_generation_worker_cls(cls) -> str: + return "vllm_omni.platforms.musa.worker.musa_generation_worker.MUSAGenerationWorker" + + @classmethod + def get_default_stage_config_path(cls) -> str: + return "vllm_omni/model_executor/stage_configs" + + @classmethod + def get_diffusion_model_impl_qualname(cls, op_name: str) -> str: + # MUSA uses default implementations for diffusion ops + if op_name == "hunyuan_fused_moe": + return "vllm_omni.diffusion.models.hunyuan_image_3.hunyuan_fused_moe.HunyuanFusedMoEDefault" + return super().get_diffusion_model_impl_qualname(op_name) + + @classmethod + def prepare_diffusion_op_runtime(cls, op_name: str, **kwargs: Any) -> None: + # MUSA uses default runtime preparation + return None + + @classmethod + def get_diffusion_attn_backend_cls( + cls, + selected_backend: str | None, + head_size: int, + ) -> str: + """Get the diffusion attention backend class path for MUSA platform. + + MUSA currently supports SDPA (Scaled Dot Product Attention) as the + primary backend. Flash Attention support may be added in future + when MUSA-specific implementations are available. + + Args: + selected_backend: User-selected backend name (e.g., "FLASH_ATTN", + "TORCH_SDPA"). If None, uses platform default. + head_size: Attention head size. + + Returns: + Fully qualified class path of the selected backend. + """ + if selected_backend is not None: + backend_upper = selected_backend.upper() + backend = DiffusionAttentionBackendEnum[backend_upper] + logger.info("Using diffusion attention backend '%s'", backend_upper) + return backend.get_path() + + # Default to SDPA for MUSA as it's the most compatible backend + logger.info("Defaulting to diffusion attention backend SDPA") + return DiffusionAttentionBackendEnum.TORCH_SDPA.get_path() + + @classmethod + def supports_torch_inductor(cls) -> bool: + """MUSA supports torch.compile with inductor backend.""" + return True + + @classmethod + def get_torch_device(cls, local_rank: int | None = None) -> torch.device: + """Get the torch device for MUSA platform. + + Args: + local_rank: Optional local rank for multi-GPU setups. + + Returns: + torch.device for MUSA GPU. + """ + if local_rank is None: + return torch.device("musa") + return torch.device("musa", local_rank) + + @classmethod + def get_device_count(cls) -> int: + """Get the number of available MUSA devices.""" + return torch.musa.device_count() + + @classmethod + def synchronize(cls) -> None: + """Synchronize all MUSA operations.""" + torch.musa.synchronize() + + @classmethod + def get_free_memory(cls, device: torch.device | None = None) -> int: + """Get the free memory on the MUSA device. + + Args: + device: Optional device to query. If None, uses current device. + + Returns: + Free memory in bytes. + """ + free, _ = torch.musa.mem_get_info(device) + return free + + @classmethod + def get_device_name(cls, device_id: int = 0) -> str: + return torch.musa.get_device_name(device_id) diff --git a/vllm_omni/platforms/musa/worker/__init__.py b/vllm_omni/platforms/musa/worker/__init__.py new file mode 100644 index 000000000..bd0054870 --- /dev/null +++ b/vllm_omni/platforms/musa/worker/__init__.py @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +from vllm_omni.platforms.musa.worker.musa_ar_worker import MUSAARWorker +from vllm_omni.platforms.musa.worker.musa_generation_worker import ( + MUSAGenerationWorker, +) + +__all__ = ["MUSAARWorker", "MUSAGenerationWorker"] diff --git a/vllm_omni/platforms/musa/worker/musa_ar_worker.py b/vllm_omni/platforms/musa/worker/musa_ar_worker.py new file mode 100644 index 000000000..258e911df --- /dev/null +++ b/vllm_omni/platforms/musa/worker/musa_ar_worker.py @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""MUSA AR (Autoregressive) worker for vLLM-Omni. + +This worker handles autoregressive model stages (thinker/talker) on MUSA devices. +""" + +import gc +import os + +import torch +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.utils.mem_utils import MemorySnapshot, format_gib +from vllm.utils.torch_utils import set_random_seed +from vllm.v1.utils import report_usage_stats +from vllm.v1.worker.gpu_worker import init_worker_distributed_environment +from vllm.v1.worker.utils import request_memory +from vllm.v1.worker.workspace import init_workspace_manager + +from vllm_omni.worker.base import OmniGPUWorkerBase +from vllm_omni.worker.gpu_ar_model_runner import GPUARModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin + +logger = init_logger(__name__) + + +class MUSAARWorker(OmniWorkerMixin, OmniGPUWorkerBase): + """MUSA AR worker for thinker/talker stages in Omni model.""" + + def init_device(self): + """Initialize the MUSA device for this worker.""" + # This env var set by Ray causes exceptions with graph building. + os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None) + parallel_config = self.parallel_config + if ( + parallel_config.distributed_executor_backend not in ("ray", "external_launcher") + and parallel_config.data_parallel_backend != "ray" + and parallel_config.nnodes_within_dp == 1 + ): + # Use local DP rank if available, otherwise use global DP rank. + dp_local_rank = self.parallel_config.data_parallel_rank_local + if dp_local_rank is None: + dp_local_rank = self.parallel_config.data_parallel_index + + tp_pp_world_size = self.parallel_config.pipeline_parallel_size * self.parallel_config.tensor_parallel_size + + # DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK + self.local_rank += dp_local_rank * tp_pp_world_size + assert self.local_rank < torch.musa.device_count(), ( + f"DP adjusted local rank {self.local_rank} is out of bounds. " + ) + visible_device_count = torch.musa.device_count() + assert self.parallel_config.local_world_size <= visible_device_count, ( + f"local_world_size ({self.parallel_config.local_world_size}) must " + f"be less than or equal to the number of visible devices " + f"({visible_device_count})." + ) + + self.device = torch.device(f"musa:{self.local_rank}") + torch.musa.set_device(self.device) + + current_platform.check_if_supports_dtype(self.model_config.dtype) + + # Initialize the distributed environment BEFORE taking memory snapshot + # This ensures NCCL buffers are allocated before we measure available memory + init_worker_distributed_environment( + self.vllm_config, + self.rank, + self.distributed_init_method, + self.local_rank, + current_platform.dist_backend, + ) + + # Set random seed. + set_random_seed(self.model_config.seed) + + # Now take memory snapshot after distributed environment is initialized + gc.collect() + torch.musa.empty_cache() + + # Take current memory snapshot + self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device) + self.requested_memory = request_memory(init_snapshot, self.cache_config) + logger.debug("worker init memory snapshot: %r", self.init_snapshot) + logger.debug("worker requested memory: %sGiB", format_gib(self.requested_memory)) + + # Initialize workspace manager + num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1 + init_workspace_manager(self.device, num_ubatches) + + if self.use_v2_model_runner: + # OMNI: v2 model runner does not yet include omni hooks. + logger.warning("OMNI MUSAARWorker forces v1 model runner for omni hooks.") + self.use_v2_model_runner = False + + # Construct the model runner + self.model_runner = GPUARModelRunner(self.vllm_config, self.device) + + if self.rank == 0: + # If usage stat is enabled, collect relevant info. + report_usage_stats(self.vllm_config) diff --git a/vllm_omni/platforms/musa/worker/musa_generation_worker.py b/vllm_omni/platforms/musa/worker/musa_generation_worker.py new file mode 100644 index 000000000..f433f8897 --- /dev/null +++ b/vllm_omni/platforms/musa/worker/musa_generation_worker.py @@ -0,0 +1,106 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +"""MUSA generation worker for vLLM-Omni. + +This worker handles non-autoregressive generation stages (e.g., code2wav waveform +generation) on MUSA devices. +""" + +import gc +import os + +import torch +from vllm.logger import init_logger +from vllm.platforms import current_platform +from vllm.tracing import instrument +from vllm.utils.mem_utils import MemorySnapshot, format_gib +from vllm.utils.torch_utils import set_random_seed +from vllm.v1.utils import report_usage_stats +from vllm.v1.worker.gpu_worker import init_worker_distributed_environment +from vllm.v1.worker.utils import request_memory +from vllm.v1.worker.workspace import init_workspace_manager + +from vllm_omni.worker.base import OmniGPUWorkerBase +from vllm_omni.worker.gpu_generation_model_runner import GPUGenerationModelRunner +from vllm_omni.worker.mixins import OmniWorkerMixin + +logger = init_logger(__name__) + + +class MUSAGenerationWorker(OmniWorkerMixin, OmniGPUWorkerBase): + """MUSA generation worker for non-AR waveform generation stage.""" + + @instrument(span_name="Init device") + def init_device(self): + """Initialize the MUSA device for this worker.""" + # This env var set by Ray causes exceptions with graph building. + os.environ.pop("NCCL_ASYNC_ERROR_HANDLING", None) + parallel_config = self.parallel_config + if ( + parallel_config.distributed_executor_backend not in ("ray", "external_launcher") + and parallel_config.data_parallel_backend != "ray" + and parallel_config.nnodes_within_dp == 1 + ): + # Use local DP rank if available, otherwise use global DP rank. + dp_local_rank = self.parallel_config.data_parallel_rank_local + if dp_local_rank is None: + dp_local_rank = self.parallel_config.data_parallel_index + + tp_pp_world_size = self.parallel_config.pipeline_parallel_size * self.parallel_config.tensor_parallel_size + + # DP_LOCAL_RANK * TP_PP_WORLD_SIZE + TP_LOCAL_RANK + self.local_rank += dp_local_rank * tp_pp_world_size + assert self.local_rank < torch.musa.device_count(), ( + f"DP adjusted local rank {self.local_rank} is out of bounds. " + ) + visible_device_count = torch.musa.device_count() + assert self.parallel_config.local_world_size <= visible_device_count, ( + f"local_world_size ({self.parallel_config.local_world_size}) must " + f"be less than or equal to the number of visible devices " + f"({visible_device_count})." + ) + + self.device = torch.device(f"musa:{self.local_rank}") + torch.musa.set_device(self.device) + + current_platform.check_if_supports_dtype(self.model_config.dtype) + + # Initialize the distributed environment BEFORE taking memory snapshot + # This ensures NCCL buffers are allocated before we measure available memory + init_worker_distributed_environment( + self.vllm_config, + self.rank, + self.distributed_init_method, + self.local_rank, + current_platform.dist_backend, + ) + + # Set random seed. + set_random_seed(self.model_config.seed) + + # Now take memory snapshot after distributed environment is initialized + gc.collect() + torch.musa.empty_cache() + + # Take current memory snapshot + self.init_snapshot = init_snapshot = MemorySnapshot(device=self.device) + self.requested_memory = request_memory(init_snapshot, self.cache_config) + logger.debug("worker init memory snapshot: %r", self.init_snapshot) + logger.debug("worker requested memory: %sGiB", format_gib(self.requested_memory)) + + # Initialize workspace manager + num_ubatches = 2 if self.vllm_config.parallel_config.enable_dbo else 1 + init_workspace_manager(self.device, num_ubatches) + + if self.use_v2_model_runner: + # OMNI: v2 model runner does not yet include omni hooks. + logger.warning("OMNI MUSAGenerationWorker forces v1 model runner for omni hooks.") + self.use_v2_model_runner = False + + # Construct the model runner + self.model_runner = GPUGenerationModelRunner(self.vllm_config, self.device) + + if self.rank == 0: + # If usage stat is enabled, collect relevant info. + report_usage_stats(self.vllm_config) diff --git a/vllm_omni/profiler/omni_torch_profiler.py b/vllm_omni/profiler/omni_torch_profiler.py index 7d03ad328..2257a2128 100644 --- a/vllm_omni/profiler/omni_torch_profiler.py +++ b/vllm_omni/profiler/omni_torch_profiler.py @@ -18,11 +18,12 @@ from vllm.profiler.wrapper import WorkerProfiler logger = init_logger(__name__) # NPU has its custom profiler -TorchProfilerActivity = Literal["CPU", "CUDA", "XPU", "NPU"] +TorchProfilerActivity = Literal["CPU", "CUDA", "XPU", "NPU", "MUSA"] TorchProfilerActivityMap = { "CPU": torch.profiler.ProfilerActivity.CPU, "CUDA": torch.profiler.ProfilerActivity.CUDA, "XPU": torch.profiler.ProfilerActivity.XPU, + "MUSA": torch.profiler.ProfilerActivity.CUDA, }