{ "add_bos_token": false, "add_eos_token": false, "added_tokens_decoder": { "0": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "1": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "2": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "3": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": true }, "50": { "content": "<|im_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "51": { "content": "<|im_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "52": { "content": "<|object_ref_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "53": { "content": "<|object_ref_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "54": { "content": "<|box_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "55": { "content": "<|box_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "56": { "content": "<|quad_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "57": { "content": "<|quad_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "58": { "content": "<|vision_start|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "59": { "content": "<|vision_end|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "60": { "content": "<|vision_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "61": { "content": "<|image_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "62": { "content": "<|video_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "63": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "64": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "65": { "content": "<|fim_prefix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "66": { "content": "<|fim_middle|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "67": { "content": "<|fim_suffix|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "68": { "content": "<|fim_pad|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "69": { "content": "<|repo_name|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "70": { "content": "<|file_sep|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": false }, "71": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "72": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "73": { "content": "", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "74": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "75": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "76": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "77": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "78": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "79": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "80": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "81": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "82": { "content": "<|im_sep|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "83": { "content": "<|tool_call|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "84": { "content": "<|arguments|>", "lstrip": false, "normalized": false, "rstrip": false, "single_word": false, "special": true }, "85": { "content": "<|o1_step|>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "86": { "content": "<|o1_answer|>", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "87": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false }, "88": { "content": "", "lstrip": false, "normalized": true, "rstrip": false, "single_word": false, "special": false } }, "additional_special_tokens": [ "<|im_start|>", "<|im_end|>", "<|object_ref_start|>", "<|object_ref_end|>", "<|box_start|>", "<|box_end|>", "<|quad_start|>", "<|quad_end|>", "<|vision_start|>", "<|vision_end|>", "<|vision_pad|>", "<|image_pad|>", "<|video_pad|>", "", "", "", "", "<|im_sep|>", "<|tool_call|>", "<|arguments|>" ], "auto_map": { "AutoTokenizer": [ "tokenization_baichuan.BaichuanTokenizer", null ] }, "bos_token": "", "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'' + message['content']}}{% elif message['role'] == 'user_system' %}{{'' + message['content']}}{% elif message['role'] == 'user' %}{{'' + message['content']}}{% elif message['role'] == 'assistant' %}{{'' + message['content']}}{% elif message['role'] == 'function' %}{{'' + message['content']}}{% elif message['role'] == 'code' %}{{'' + message['content']}}{% else %}{{ raise_exception('Invalid message role: ' + message['role']) }}{% endif %}{% endfor %}{% if add_generation_prompt %}{{''}}{% endif %}", "clean_up_tokenization_spaces": false, "eos_token": "", "extra_special_tokens": {}, "model_max_length": 32768, "pad_token": "", "sp_model_kwargs": {}, "tokenizer_class": "BaichuanTokenizer", "unk_token": "", "use_fast": false }