from transformers import PreTrainedTokenizerFast tokenizer = PreTrainedTokenizerFast( tokenizer_file="tokenizer.json", unk_token="", pad_token="", cls_token="", sep_token="", mask_token="" )