[Hardware] Support platforms and plugin system (#774)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
b11d436e19
commit
c9392dff01
103 files changed
+1770
-1218
No files matched your search
@@ -36,7 +36,7 @@ from vllm_omni.model_executor.model_loader.weight_utils import download_weights_
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges
|
||||
from vllm_omni.model_executor.models.vision import get_llm_pos_ids_for_vision
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
TALKER_CODEC_EOS_TOKEN_ID = 8294
|
||||
TALKER_CODEC_BOS_TOKEN_ID = 8293
|
||||
@@ -247,7 +247,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
if inputs_embeds is not None and inputs_embeds.device != thinker_dev:
|
||||
inputs_embeds = inputs_embeds.to(thinker_dev)
|
||||
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# TODO: remove this hack when NPU supports batched inputs properly
|
||||
thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
|
||||
# For MRoPE, positions shape is [3, num_tokens] (T/H/W), don't slice it
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.model_executor.models.qwen2_5_omni.audio_length import cap_and_align_mel_length, resolve_max_mel_frames
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
|
||||
# Provide a no-op auto_docstring decorator to satisfy annotations if missing
|
||||
@@ -732,7 +732,7 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) ->
|
||||
beta = 0.0
|
||||
|
||||
# TODO: When torch.kaiser_window supports NPU, remove the device="cpu" argument
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
kaiser_window = torch.kaiser_window(
|
||||
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
|
||||
).to("npu")
|
||||
@@ -796,7 +796,7 @@ class UpSample1d(nn.Module):
|
||||
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# TODO: When F.pad supports replicate mode on NPU, remove this branch
|
||||
input_dtype = hidden_states.dtype
|
||||
# F.pad in NPU doesn't support BF16 when mode is replicate.
|
||||
@@ -843,7 +843,7 @@ class DownSample1d(nn.Module):
|
||||
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
input_dtype = hidden_states.dtype
|
||||
# F.pad in NPU doesn't support BF16 when mode is replicate.
|
||||
# To ensure the accuracy, manually pad the input tensor.
|
||||
|
||||
@@ -35,7 +35,7 @@ from vllm.v1.sample.sampler import Sampler
|
||||
from vllm_omni.model_executor.custom_process_mixin import CustomProcessMixin
|
||||
from vllm_omni.model_executor.models.output_templates import OmniOutput
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, safe_tensor_reshape
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
from vllm_omni.platforms import current_omni_platform
|
||||
|
||||
# Special token IDs for Qwen3 Omni MoE
|
||||
# Reference: https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
|
||||
@@ -269,7 +269,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
# ========== Stage 1: Thinker ==========
|
||||
if self.model_stage == "thinker":
|
||||
thinker_dev = self._module_device(self.thinker)
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# Normalize to batched inputs if needed
|
||||
_added_batch_dim = False
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
@@ -308,7 +308,7 @@ class Qwen3OmniMoeForConditionalGeneration(
|
||||
"capture_layer_indices": [0, int(accept_layer)],
|
||||
"return_hidden_states": True,
|
||||
}
|
||||
if is_npu():
|
||||
if current_omni_platform.is_npu():
|
||||
# TODO: remove this hack when NPU supports batched inputs properly
|
||||
thinker_input_ids = input_ids[0] if input_ids is not None and _added_batch_dim else input_ids
|
||||
thinker_inputs_embeds = (
|
||||
|
||||
Reference in new issue
Block a user