gpt-4-tokenizer-sc-tokens / added_tokens.json
RaymondLi's picture
Upload tokenizer
90cd208
raw
history blame contribute delete
626 Bytes
{
"<commit_after>": 100279,
"<commit_before>": 100292,
"<commit_msg>": 100286,
"<empty_output>": 100293,
"<filename>": 100283,
"<fim_middle>": 100281,
"<fim_pad>": 100287,
"<fim_prefix>": 100285,
"<fim_suffix>": 100284,
"<gh_stars>": 100277,
"<issue_closed>": 100291,
"<issue_comment>": 100288,
"<issue_start>": 100278,
"<jupyter_code>": 100282,
"<jupyter_output>": 100294,
"<jupyter_start>": 100290,
"<jupyter_text>": 100280,
"<reponame>": 100289,
"<|endofprompt|>": 100276,
"<|endoftext|>": 100257,
"<|fim_middle|>": 100259,
"<|fim_prefix|>": 100258,
"<|fim_suffix|>": 100260
}