alvarobartt HF Staff commited on
Commit
2518e09
·
verified ·
1 Parent(s): 57c6dc7

Add `AutoTokenizer.from_pretrained` export

Browse files

```python
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("jinaai/jina-code-embeddings-1.5b")
tokenizer.save_pretrained()
```

.gitattributes CHANGED
@@ -1 +1,2 @@
1
  *.safetensors filter=lfs diff=lfs merge=lfs -text
 
 
1
  *.safetensors filter=lfs diff=lfs merge=lfs -text
2
+ tokenizer.json filter=lfs diff=lfs merge=lfs -text
added_tokens.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ {
2
+ "<|endoftext|>": 151643
3
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,23 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "eos_token": {
3
+ "content": "<|endoftext|>",
4
+ "lstrip": false,
5
+ "normalized": false,
6
+ "rstrip": false,
7
+ "single_word": false
8
+ },
9
+ "pad_token": {
10
+ "content": "<|endoftext|>",
11
+ "lstrip": false,
12
+ "normalized": false,
13
+ "rstrip": false,
14
+ "single_word": false
15
+ },
16
+ "unk_token": {
17
+ "content": "<|endoftext|>",
18
+ "lstrip": false,
19
+ "normalized": false,
20
+ "rstrip": false,
21
+ "single_word": false
22
+ }
23
+ }
tokenizer.json ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f0c3c6d1699d808eea431043c5d9db4e6c191698789c69a5d35f11aeabc2304a
3
+ size 11417884
tokenizer_config.json ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "added_tokens_decoder": {
3
+ "151643": {
4
+ "content": "<|endoftext|>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false,
9
+ "special": true
10
+ }
11
+ },
12
+ "bos_token": null,
13
+ "clean_up_tokenization_spaces": false,
14
+ "eos_token": "<|endoftext|>",
15
+ "errors": "replace",
16
+ "extra_special_tokens": {},
17
+ "model_max_length": 1000000000000000019884624838656,
18
+ "pad_token": "<|endoftext|>",
19
+ "split_special_tokens": false,
20
+ "tokenizer_class": "Qwen2Tokenizer",
21
+ "unk_token": "<|endoftext|>"
22
+ }