10f998c
1
2
3
4
5
6
7
8
9
10
from transformers import PreTrainedTokenizerFast tokenizer = PreTrainedTokenizerFast( tokenizer_file="tokenizer.json", unk_token="<unk>", pad_token="<pad>", cls_token="<cls>", sep_token="<sep>", mask_token="<mask>" )
