legendhasit
/

xgen-7b-8k-open-instruct-8bit

Text Generation

text-generation-inference

Inference Endpoints

8-bit precision

Model card Files Files and versions Community

froggoboom commited on Jul 31, 2023

Commit

5d93c13

·

1 Parent(s): ee52311

Update tokenizer_config.json

Files changed (1) hide show

tokenizer_config.json +2 -3

tokenizer_config.json CHANGED Viewed

@@ -3,7 +3,7 @@
   "add_special_tokens": true,
   "auto_map": {
     "AutoTokenizer": [
-      "VMware/xgen-7b-8k-open-instruct--tokenization_xgen.XgenTokenizer",
       null
     ]
   },
@@ -16,8 +16,7 @@
     "rstrip": false,
     "single_word": false
   },
-  "model_max_length": 8192,
   "pad_token": null,
-  "return_token_type_ids": false,
   "tokenizer_class": "XgenTokenizer"
 }

   "add_special_tokens": true,
   "auto_map": {
     "AutoTokenizer": [
+      "tokenization_xgen.XgenTokenizer",
       null
     ]
   },
     "rstrip": false,
     "single_word": false
   },
+  "model_max_length": 1000000000000000019884624838656,
   "pad_token": null,
   "tokenizer_class": "XgenTokenizer"
 }