dsinghviandHongsheng Liu
b63e7c7c9a
Signed-off-by: Divyansh Singhvi <divyanshsinghvi@gmail.com> Signed-off-by: dsinghvi <divyanshsinghvi@gmail.com> Co-authored-by: Hongsheng Liu <liuhongsheng4@huawei.com>
357 lines
9.5 KiB
Python
357 lines
9.5 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from functools import cache
|
|
|
|
import torch
|
|
from transformers import AutoTokenizer
|
|
|
|
|
|
class CosyVoice2Tokenizer:
|
|
def __init__(self, token_path, skip_special_tokens=True):
|
|
super().__init__()
|
|
# NOTE: non-chat model, all these special tokens keep randomly initialized.
|
|
special_tokens = {
|
|
"eos_token": "<|endoftext|>",
|
|
"pad_token": "<|endoftext|>",
|
|
"additional_special_tokens": [
|
|
"<|im_start|>",
|
|
"<|im_end|>",
|
|
"<|endofprompt|>",
|
|
"[breath]",
|
|
"<strong>",
|
|
"</strong>",
|
|
"[noise]",
|
|
"[laughter]",
|
|
"[cough]",
|
|
"[clucking]",
|
|
"[accent]",
|
|
"[quick_breath]",
|
|
"<laughter>",
|
|
"</laughter>",
|
|
"[hissing]",
|
|
"[sigh]",
|
|
"[vocalized-noise]",
|
|
"[lipsmack]",
|
|
"[mn]",
|
|
],
|
|
}
|
|
self.special_tokens = special_tokens
|
|
self.tokenizer = AutoTokenizer.from_pretrained(token_path)
|
|
self.tokenizer.add_special_tokens(special_tokens)
|
|
self.skip_special_tokens = skip_special_tokens
|
|
|
|
def encode(self, text, **kwargs):
|
|
tokens = self.tokenizer([text], return_tensors="pt")
|
|
tokens = tokens["input_ids"][0].cpu().tolist()
|
|
return tokens
|
|
|
|
def decode(self, tokens):
|
|
tokens = torch.tensor(tokens, dtype=torch.int64)
|
|
text = self.tokenizer.batch_decode([tokens], skip_special_tokens=self.skip_special_tokens)[0]
|
|
return text
|
|
|
|
|
|
class CosyVoice3Tokenizer(CosyVoice2Tokenizer):
|
|
def __init__(self, token_path, skip_special_tokens=True):
|
|
# NOTE: non-chat model, all these special tokens keep randomly initialized.
|
|
special_tokens = {
|
|
"eos_token": "<|endoftext|>",
|
|
"pad_token": "<|endoftext|>",
|
|
"additional_special_tokens": [
|
|
"<|im_start|>",
|
|
"<|im_end|>",
|
|
"<|endofprompt|>",
|
|
"[breath]",
|
|
"<strong>",
|
|
"</strong>",
|
|
"[noise]",
|
|
"[laughter]",
|
|
"[cough]",
|
|
"[clucking]",
|
|
"[accent]",
|
|
"[quick_breath]",
|
|
"<laughter>",
|
|
"</laughter>",
|
|
"[hissing]",
|
|
"[sigh]",
|
|
"[vocalized-noise]",
|
|
"[lipsmack]",
|
|
"[mn]",
|
|
"<|endofsystem|>",
|
|
"[AA]",
|
|
"[AA0]",
|
|
"[AA1]",
|
|
"[AA2]",
|
|
"[AE]",
|
|
"[AE0]",
|
|
"[AE1]",
|
|
"[AE2]",
|
|
"[AH]",
|
|
"[AH0]",
|
|
"[AH1]",
|
|
"[AH2]",
|
|
"[AO]",
|
|
"[AO0]",
|
|
"[AO1]",
|
|
"[AO2]",
|
|
"[AW]",
|
|
"[AW0]",
|
|
"[AW1]",
|
|
"[AW2]",
|
|
"[AY]",
|
|
"[AY0]",
|
|
"[AY1]",
|
|
"[AY2]",
|
|
"[B]",
|
|
"[CH]",
|
|
"[D]",
|
|
"[DH]",
|
|
"[EH]",
|
|
"[EH0]",
|
|
"[EH1]",
|
|
"[EH2]",
|
|
"[ER]",
|
|
"[ER0]",
|
|
"[ER1]",
|
|
"[ER2]",
|
|
"[EY]",
|
|
"[EY0]",
|
|
"[EY1]",
|
|
"[EY2]",
|
|
"[F]",
|
|
"[G]",
|
|
"[HH]",
|
|
"[IH]",
|
|
"[IH0]",
|
|
"[IH1]",
|
|
"[IH2]",
|
|
"[IT]",
|
|
"[IT0]",
|
|
"[IT1]",
|
|
"[IT2]",
|
|
"[JH]",
|
|
"[K]",
|
|
"[L]",
|
|
"[M]",
|
|
"[N]",
|
|
"[NG]",
|
|
"[OW]",
|
|
"[OW0]",
|
|
"[OW1]",
|
|
"[OW2]",
|
|
"[OY]",
|
|
"[OY0]",
|
|
"[OY1]",
|
|
"[OY2]",
|
|
"[P]",
|
|
"[R]",
|
|
"[S]",
|
|
"[SH]",
|
|
"[T]",
|
|
"[TH]",
|
|
"[UH]",
|
|
"[UH0]",
|
|
"[UH1]",
|
|
"[UH2]",
|
|
"[UW]",
|
|
"[UW0]",
|
|
"[UW1]",
|
|
"[UW2]",
|
|
"[V]",
|
|
"[W]",
|
|
"[Y]",
|
|
"[Z]",
|
|
"[ZH]",
|
|
"[a]",
|
|
"[ai]",
|
|
"[an]",
|
|
"[ang]",
|
|
"[ao]",
|
|
"[b]",
|
|
"[c]",
|
|
"[ch]",
|
|
"[d]",
|
|
"[e]",
|
|
"[ei]",
|
|
"[en]",
|
|
"[eng]",
|
|
"[f]",
|
|
"[g]",
|
|
"[h]",
|
|
"[i]",
|
|
"[ian]",
|
|
"[in]",
|
|
"[ing]",
|
|
"[iu]",
|
|
"[ià]",
|
|
"[iàn]",
|
|
"[iàng]",
|
|
"[iào]",
|
|
"[iá]",
|
|
"[ián]",
|
|
"[iáng]",
|
|
"[iáo]",
|
|
"[iè]",
|
|
"[ié]",
|
|
"[iòng]",
|
|
"[ióng]",
|
|
"[iù]",
|
|
"[iú]",
|
|
"[iā]",
|
|
"[iān]",
|
|
"[iāng]",
|
|
"[iāo]",
|
|
"[iē]",
|
|
"[iě]",
|
|
"[iōng]",
|
|
"[iū]",
|
|
"[iǎ]",
|
|
"[iǎn]",
|
|
"[iǎng]",
|
|
"[iǎo]",
|
|
"[iǒng]",
|
|
"[iǔ]",
|
|
"[j]",
|
|
"[k]",
|
|
"[l]",
|
|
"[m]",
|
|
"[n]",
|
|
"[o]",
|
|
"[ong]",
|
|
"[ou]",
|
|
"[p]",
|
|
"[q]",
|
|
"[r]",
|
|
"[s]",
|
|
"[sh]",
|
|
"[t]",
|
|
"[u]",
|
|
"[uang]",
|
|
"[ue]",
|
|
"[un]",
|
|
"[uo]",
|
|
"[uà]",
|
|
"[uài]",
|
|
"[uàn]",
|
|
"[uàng]",
|
|
"[uá]",
|
|
"[uái]",
|
|
"[uán]",
|
|
"[uáng]",
|
|
"[uè]",
|
|
"[ué]",
|
|
"[uì]",
|
|
"[uí]",
|
|
"[uò]",
|
|
"[uó]",
|
|
"[uā]",
|
|
"[uāi]",
|
|
"[uān]",
|
|
"[uāng]",
|
|
"[uē]",
|
|
"[uě]",
|
|
"[uī]",
|
|
"[uō]",
|
|
"[uǎ]",
|
|
"[uǎi]",
|
|
"[uǎn]",
|
|
"[uǎng]",
|
|
"[uǐ]",
|
|
"[uǒ]",
|
|
"[vè]",
|
|
"[w]",
|
|
"[x]",
|
|
"[y]",
|
|
"[z]",
|
|
"[zh]",
|
|
"[à]",
|
|
"[ài]",
|
|
"[àn]",
|
|
"[àng]",
|
|
"[ào]",
|
|
"[á]",
|
|
"[ái]",
|
|
"[án]",
|
|
"[áng]",
|
|
"[áo]",
|
|
"[è]",
|
|
"[èi]",
|
|
"[èn]",
|
|
"[èng]",
|
|
"[èr]",
|
|
"[é]",
|
|
"[éi]",
|
|
"[én]",
|
|
"[éng]",
|
|
"[ér]",
|
|
"[ì]",
|
|
"[ìn]",
|
|
"[ìng]",
|
|
"[í]",
|
|
"[ín]",
|
|
"[íng]",
|
|
"[ò]",
|
|
"[òng]",
|
|
"[òu]",
|
|
"[ó]",
|
|
"[óng]",
|
|
"[óu]",
|
|
"[ù]",
|
|
"[ùn]",
|
|
"[ú]",
|
|
"[ún]",
|
|
"[ā]",
|
|
"[āi]",
|
|
"[ān]",
|
|
"[āng]",
|
|
"[āo]",
|
|
"[ē]",
|
|
"[ēi]",
|
|
"[ēn]",
|
|
"[ēng]",
|
|
"[ě]",
|
|
"[ěi]",
|
|
"[ěn]",
|
|
"[ěng]",
|
|
"[ěr]",
|
|
"[ī]",
|
|
"[īn]",
|
|
"[īng]",
|
|
"[ō]",
|
|
"[ōng]",
|
|
"[ōu]",
|
|
"[ū]",
|
|
"[ūn]",
|
|
"[ǎ]",
|
|
"[ǎi]",
|
|
"[ǎn]",
|
|
"[ǎng]",
|
|
"[ǎo]",
|
|
"[ǐ]",
|
|
"[ǐn]",
|
|
"[ǐng]",
|
|
"[ǒ]",
|
|
"[ǒng]",
|
|
"[ǒu]",
|
|
"[ǔ]",
|
|
"[ǔn]",
|
|
"[ǘ]",
|
|
"[ǚ]",
|
|
"[ǜ]",
|
|
],
|
|
}
|
|
self.special_tokens = special_tokens
|
|
self.tokenizer = AutoTokenizer.from_pretrained(token_path)
|
|
self.tokenizer.add_special_tokens(special_tokens)
|
|
self.skip_special_tokens = skip_special_tokens
|
|
|
|
|
|
@cache
|
|
def get_qwen_tokenizer(token_path: str, skip_special_tokens: bool, version: str = "cosyvoice3"):
|
|
if version == "cosyvoice2":
|
|
return CosyVoice2Tokenizer(token_path=token_path, skip_special_tokens=skip_special_tokens)
|
|
elif version == "cosyvoice3":
|
|
return CosyVoice3Tokenizer(token_path=token_path, skip_special_tokens=skip_special_tokens)
|
|
else:
|
|
raise ValueError
|