[Feat] Add NPU Backend support for vLLM-Omni (#89)
Signed-off-by: gcanlin <canlinguosdu@gmail.com>
This commit is contained in:
1 parent
88c00fb271
commit
ec9e04027d
25 files changed
+2932
-85
No files matched your search
@@ -35,6 +35,7 @@ from vllm_omni.model_executor.models.qwen2_5_omni.qwen2_5_omni_thinker import (
|
||||
)
|
||||
from vllm_omni.model_executor.models.utils import add_prefix_to_loaded_weights, split_list_into_ranges
|
||||
from vllm_omni.model_executor.models.vision import get_llm_pos_ids_for_vision
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
|
||||
TALKER_CODEC_EOS_TOKEN_ID = 8294
|
||||
TALKER_CODEC_BOS_TOKEN_ID = 8293
|
||||
@@ -201,6 +202,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
"""
|
||||
if self.model_stage == "thinker":
|
||||
# Normalize to batched inputs if caller provides 1D/2D unbatched tensors
|
||||
# TODO: Remove this hack when NPU supports batched inputs properly
|
||||
added_batch_dim = False
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
input_ids = input_ids.unsqueeze(0)
|
||||
@@ -228,12 +230,25 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
positions = positions.to(thinker_dev)
|
||||
if inputs_embeds is not None and inputs_embeds.device != thinker_dev:
|
||||
inputs_embeds = inputs_embeds.to(thinker_dev)
|
||||
|
||||
if is_npu():
|
||||
# TODO: remove this hack when NPU supports batched inputs properly
|
||||
thinker_input_ids = input_ids[0] if input_ids is not None and added_batch_dim else input_ids
|
||||
thinker_positions = positions[0] if positions.ndim > 1 else positions
|
||||
thinker_inputs_embeds = (
|
||||
inputs_embeds[0] if inputs_embeds is not None and added_batch_dim else inputs_embeds
|
||||
)
|
||||
else:
|
||||
thinker_input_ids = input_ids
|
||||
thinker_positions = positions[0]
|
||||
thinker_inputs_embeds = inputs_embeds
|
||||
|
||||
# Run thinker
|
||||
thinker_output = self.thinker(
|
||||
input_ids=input_ids,
|
||||
positions=positions[0],
|
||||
input_ids=thinker_input_ids,
|
||||
positions=thinker_positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
inputs_embeds=thinker_inputs_embeds,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
@@ -851,7 +866,7 @@ class Qwen2_5OmniForConditionalGeneration(
|
||||
__init__."""
|
||||
if self.token2wav is None or self.token2wav_config is None:
|
||||
return
|
||||
device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
device = self._module_device(self.token2wav)
|
||||
# optional speaker resources
|
||||
conds = getattr(self.token2wav_config, "conds", None)
|
||||
ref_mels = getattr(self.token2wav_config, "ref_mels", None)
|
||||
|
||||
@@ -33,6 +33,8 @@ from vllm.v1.outputs import SamplerOutput
|
||||
from vllm.v1.sample.metadata import SamplingMetadata
|
||||
from vllm.v1.sample.sampler import Sampler
|
||||
|
||||
from vllm_omni.utils.platform_utils import is_npu
|
||||
|
||||
|
||||
# Provide a no-op auto_docstring decorator to satisfy annotations if missing
|
||||
def auto_docstring(func=None, **_kwargs):
|
||||
@@ -724,7 +726,13 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) ->
|
||||
else:
|
||||
beta = 0.0
|
||||
|
||||
kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32)
|
||||
# TODO: When torch.kaiser_window supports NPU, remove the device="cpu" argument
|
||||
if is_npu():
|
||||
kaiser_window = torch.kaiser_window(
|
||||
kernel_size, beta=beta, periodic=False, dtype=torch.float32, device="cpu"
|
||||
).to("npu")
|
||||
else:
|
||||
kaiser_window = torch.kaiser_window(kernel_size, beta=beta, periodic=False, dtype=torch.float32)
|
||||
|
||||
# Compute time indices
|
||||
if is_even:
|
||||
@@ -734,7 +742,7 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) ->
|
||||
|
||||
# Compute sinc filter
|
||||
if cutoff == 0:
|
||||
return torch.zeros((1, 1, kernel_size), dtype=torch.float32) # Ensures correct shape
|
||||
return torch.zeros((1, 1, kernel_size), dtype=torch.float32)
|
||||
|
||||
sinc_filter = torch.sinc(2 * cutoff * time_indices)
|
||||
normalized_filter = 2 * cutoff * kaiser_window * sinc_filter
|
||||
@@ -745,6 +753,29 @@ def kaiser_sinc_filter1d(cutoff: float, half_width: float, kernel_size: int) ->
|
||||
return normalized_filter.view(1, 1, kernel_size)
|
||||
|
||||
|
||||
def replication_pad_1d(hidden_states: torch.Tensor, pad_left: int, pad_right: int) -> torch.Tensor:
|
||||
"""
|
||||
Manual replicate padding to avoid replication_pad1d kernel limits on NPU.
|
||||
TODO: remove when F.pad supports replicate mode on NPU.
|
||||
"""
|
||||
# NOTE: a immature implementation for running in NPU. Need to discuss.
|
||||
if pad_left == 0 and pad_right == 0:
|
||||
return hidden_states
|
||||
|
||||
segments = []
|
||||
if pad_left > 0:
|
||||
left = hidden_states[..., :1].expand(*hidden_states.shape[:-1], pad_left)
|
||||
segments.append(left)
|
||||
|
||||
segments.append(hidden_states)
|
||||
|
||||
if pad_right > 0:
|
||||
right = hidden_states[..., -1:].expand(*hidden_states.shape[:-1], pad_right)
|
||||
segments.append(right)
|
||||
|
||||
return torch.cat(segments, dim=-1)
|
||||
|
||||
|
||||
class UpSample1d(nn.Module):
|
||||
def __init__(self, ratio=2, kernel_size=None):
|
||||
super().__init__()
|
||||
@@ -760,14 +791,28 @@ class UpSample1d(nn.Module):
|
||||
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
hidden_states_dtype = hidden_states.dtype
|
||||
hidden_states = F.pad(hidden_states, (self.pad, self.pad), mode="replicate").to(self.filter.dtype)
|
||||
hidden_states = self.ratio * F.conv_transpose1d(
|
||||
hidden_states,
|
||||
self.filter.expand(channels, -1, -1),
|
||||
stride=self.stride,
|
||||
groups=channels,
|
||||
).to(hidden_states_dtype)
|
||||
if is_npu():
|
||||
# TODO: When F.pad supports replicate mode on NPU, remove this branch
|
||||
input_dtype = hidden_states.dtype
|
||||
# F.pad in NPU doesn't support BF16 when mode is replicate.
|
||||
# To ensure the accuracy, manually pad the input tensor.
|
||||
hidden_states = replication_pad_1d(hidden_states.to(self.filter.dtype), self.pad, self.pad)
|
||||
filter_convert_dtype = self.filter.to(hidden_states.dtype)
|
||||
hidden_states = self.ratio * F.conv_transpose1d(
|
||||
hidden_states,
|
||||
filter_convert_dtype.expand(channels, -1, -1),
|
||||
stride=self.stride,
|
||||
groups=channels,
|
||||
).to(input_dtype)
|
||||
else:
|
||||
hidden_states_dtype = hidden_states.dtype
|
||||
hidden_states = F.pad(hidden_states, (self.pad, self.pad), mode="replicate").to(self.filter.dtype)
|
||||
hidden_states = self.ratio * F.conv_transpose1d(
|
||||
hidden_states,
|
||||
self.filter.expand(channels, -1, -1),
|
||||
stride=self.stride,
|
||||
groups=channels,
|
||||
).to(hidden_states_dtype)
|
||||
hidden_states = hidden_states[..., self.pad_left : -self.pad_right]
|
||||
|
||||
return hidden_states
|
||||
@@ -793,14 +838,29 @@ class DownSample1d(nn.Module):
|
||||
|
||||
def forward(self, hidden_states):
|
||||
channels = hidden_states.shape[1]
|
||||
hidden_states_dtype = hidden_states.dtype
|
||||
hidden_states = F.pad(hidden_states, (self.pad_left, self.pad_right), mode="replicate").to(self.filter.dtype)
|
||||
out = F.conv1d(
|
||||
hidden_states,
|
||||
self.filter.expand(channels, -1, -1),
|
||||
stride=self.stride,
|
||||
groups=channels,
|
||||
).to(hidden_states_dtype)
|
||||
if is_npu():
|
||||
input_dtype = hidden_states.dtype
|
||||
# F.pad in NPU doesn't support BF16 when mode is replicate.
|
||||
# To ensure the accuracy, manually pad the input tensor.
|
||||
hidden_states = replication_pad_1d(hidden_states.to(self.filter.dtype), self.pad_left, self.pad_right)
|
||||
filter_on_device = self.filter.to(device=hidden_states.device, dtype=hidden_states.dtype)
|
||||
out = F.conv1d(
|
||||
hidden_states,
|
||||
filter_on_device.expand(channels, -1, -1),
|
||||
stride=self.stride,
|
||||
groups=channels,
|
||||
).to(input_dtype)
|
||||
else:
|
||||
hidden_states_dtype = hidden_states.dtype
|
||||
hidden_states = F.pad(hidden_states, (self.pad_left, self.pad_right), mode="replicate").to(
|
||||
self.filter.dtype
|
||||
)
|
||||
out = F.conv1d(
|
||||
hidden_states,
|
||||
self.filter.expand(channels, -1, -1),
|
||||
stride=self.stride,
|
||||
groups=channels,
|
||||
).to(hidden_states_dtype)
|
||||
return out
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user