[Fix][Fish Speech] Remove redundant get_vocab() in control token encoding (#2842)
Signed-off-by: Sy03 <1370724210@qq.com>
This commit is contained in:
1 file changed
+1
-4
@@ -38,10 +38,7 @@ def _encode_plain_text(tokenizer: Any, text: str) -> list[int]:
|
||||
|
||||
|
||||
def _encode_control_token(tokenizer: Any, token: str) -> list[int]:
|
||||
vocab = tokenizer.get_vocab() if hasattr(tokenizer, "get_vocab") else {}
|
||||
token_id = vocab.get(token)
|
||||
if token_id is None:
|
||||
token_id = tokenizer.convert_tokens_to_ids(token)
|
||||
token_id = tokenizer.convert_tokens_to_ids(token)
|
||||
if token_id is None or token_id == getattr(tokenizer, "unk_token_id", None):
|
||||
raise ValueError(f"Fish Speech tokenizer is missing required control token: {token}")
|
||||
return [int(token_id)]
|
||||
|
||||
Reference in new issue
Block a user