We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent bee938d commit 5c99960Copy full SHA for 5c99960
convert-hf-to-gguf.py
@@ -266,11 +266,10 @@ def _set_vocab_gpt2(self):
266
toktypes.append(gguf.TokenType.USER_DEFINED)
267
elif reverse_vocab[i] in added_vocab:
268
tokens.append(reverse_vocab[i])
269
- if hasattr(tokenizer, "added_tokens_decoder"):
270
- if tokenizer.added_tokens_decoder[i].special:
271
- toktypes.append(gguf.TokenType.CONTROL)
272
- else:
273
- toktypes.append(gguf.TokenType.USER_DEFINED)
+ if tokenizer.added_tokens_decoder[i].special:
+ toktypes.append(gguf.TokenType.CONTROL)
+ else:
+ toktypes.append(gguf.TokenType.USER_DEFINED)
274
else:
275
276
toktypes.append(gguf.TokenType.NORMAL)
0 commit comments