teknium commited on
Commit
20776db
·
verified ·
1 Parent(s): 94710df

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +53 -53
tokenizer_config.json CHANGED
@@ -90,7 +90,7 @@
90
  "special": true
91
  },
92
  "128011": {
93
- "content": "<tools>",
94
  "lstrip": false,
95
  "normalized": false,
96
  "rstrip": false,
@@ -98,7 +98,7 @@
98
  "special": false
99
  },
100
  "128012": {
101
- "content": "</tools>",
102
  "lstrip": false,
103
  "normalized": false,
104
  "rstrip": false,
@@ -122,199 +122,199 @@
122
  "special": false
123
  },
124
  "128015": {
125
- "content": "<schema>",
126
  "lstrip": false,
127
  "normalized": false,
128
  "rstrip": false,
129
  "single_word": false,
130
- "special": false
131
  },
132
  "128016": {
133
- "content": "</schema>",
134
  "lstrip": false,
135
  "normalized": false,
136
  "rstrip": false,
137
  "single_word": false,
138
- "special": false
139
  },
140
  "128017": {
141
- "content": "<scratch_pad>",
142
  "lstrip": false,
143
  "normalized": false,
144
  "rstrip": false,
145
  "single_word": false,
146
- "special": false
147
  },
148
  "128018": {
149
- "content": "</scratch_pad>",
150
  "lstrip": false,
151
  "normalized": false,
152
  "rstrip": false,
153
  "single_word": false,
154
- "special": false
155
  },
156
  "128019": {
157
- "content": "<SCRATCHPAD>",
158
  "lstrip": false,
159
  "normalized": false,
160
  "rstrip": false,
161
  "single_word": false,
162
- "special": false
163
  },
164
  "128020": {
165
- "content": "</SCRATCHPAD>",
166
  "lstrip": false,
167
  "normalized": false,
168
  "rstrip": false,
169
  "single_word": false,
170
- "special": false
171
  },
172
  "128021": {
173
- "content": "<REASONING>",
174
  "lstrip": false,
175
  "normalized": false,
176
  "rstrip": false,
177
  "single_word": false,
178
- "special": false
179
  },
180
  "128022": {
181
- "content": "</REASONING>",
182
  "lstrip": false,
183
  "normalized": false,
184
  "rstrip": false,
185
  "single_word": false,
186
- "special": false
187
  },
188
  "128023": {
189
- "content": "<INNER_MONOLOGUE>",
190
  "lstrip": false,
191
  "normalized": false,
192
  "rstrip": false,
193
  "single_word": false,
194
- "special": false
195
  },
196
  "128024": {
197
- "content": "</INNER_MONOLOGUE>",
198
  "lstrip": false,
199
  "normalized": false,
200
  "rstrip": false,
201
  "single_word": false,
202
- "special": false
203
  },
204
  "128025": {
205
- "content": "<PLAN>",
206
  "lstrip": false,
207
  "normalized": false,
208
  "rstrip": false,
209
  "single_word": false,
210
- "special": false
211
  },
212
  "128026": {
213
- "content": "</PLAN>",
214
  "lstrip": false,
215
  "normalized": false,
216
  "rstrip": false,
217
  "single_word": false,
218
- "special": false
219
  },
220
  "128027": {
221
- "content": "<EXECUTION>",
222
  "lstrip": false,
223
  "normalized": false,
224
  "rstrip": false,
225
  "single_word": false,
226
- "special": false
227
  },
228
  "128028": {
229
- "content": "</EXECUTION>",
230
  "lstrip": false,
231
  "normalized": false,
232
  "rstrip": false,
233
  "single_word": false,
234
- "special": false
235
  },
236
  "128029": {
237
- "content": "<REFLECTION>",
238
  "lstrip": false,
239
  "normalized": false,
240
  "rstrip": false,
241
  "single_word": false,
242
- "special": false
243
  },
244
  "128030": {
245
- "content": "</REFLECTION>",
246
  "lstrip": false,
247
  "normalized": false,
248
  "rstrip": false,
249
  "single_word": false,
250
- "special": false
251
  },
252
  "128031": {
253
- "content": "<THINKING>",
254
  "lstrip": false,
255
  "normalized": false,
256
  "rstrip": false,
257
  "single_word": false,
258
- "special": false
259
  },
260
  "128032": {
261
- "content": "</THINKING>",
262
  "lstrip": false,
263
  "normalized": false,
264
  "rstrip": false,
265
  "single_word": false,
266
- "special": false
267
  },
268
  "128033": {
269
- "content": "<SOLUTION>",
270
  "lstrip": false,
271
  "normalized": false,
272
  "rstrip": false,
273
  "single_word": false,
274
- "special": false
275
  },
276
  "128034": {
277
- "content": "</SOLUTION>",
278
  "lstrip": false,
279
  "normalized": false,
280
  "rstrip": false,
281
  "single_word": false,
282
- "special": false
283
  },
284
  "128035": {
285
- "content": "<EXPLANATION>",
286
  "lstrip": false,
287
  "normalized": false,
288
  "rstrip": false,
289
  "single_word": false,
290
- "special": false
291
  },
292
  "128036": {
293
- "content": "</EXPLANATION>",
294
  "lstrip": false,
295
  "normalized": false,
296
  "rstrip": false,
297
  "single_word": false,
298
- "special": false
299
  },
300
  "128037": {
301
- "content": "<UNIT_TEST>",
302
  "lstrip": false,
303
  "normalized": false,
304
  "rstrip": false,
305
  "single_word": false,
306
- "special": false
307
  },
308
  "128038": {
309
- "content": "</UNIT_TEST>",
310
  "lstrip": false,
311
  "normalized": false,
312
  "rstrip": false,
313
  "single_word": false,
314
- "special": false
315
  },
316
  "128039": {
317
- "content": "<|im_end|>",
318
  "lstrip": false,
319
  "normalized": false,
320
  "rstrip": false,
@@ -322,12 +322,12 @@
322
  "special": true
323
  },
324
  "128040": {
325
- "content": "<|im_start|>",
326
  "lstrip": false,
327
  "normalized": false,
328
  "rstrip": false,
329
  "single_word": false,
330
- "special": false
331
  },
332
  "128041": {
333
  "content": "<|reserved_special_token_33|>",
 
90
  "special": true
91
  },
92
  "128011": {
93
+ "content": "<think>",
94
  "lstrip": false,
95
  "normalized": false,
96
  "rstrip": false,
 
98
  "special": false
99
  },
100
  "128012": {
101
+ "content": "</think>",
102
  "lstrip": false,
103
  "normalized": false,
104
  "rstrip": false,
 
122
  "special": false
123
  },
124
  "128015": {
125
+ "content": "<|reserved_special_token_7|>",
126
  "lstrip": false,
127
  "normalized": false,
128
  "rstrip": false,
129
  "single_word": false,
130
+ "special": true
131
  },
132
  "128016": {
133
+ "content": "<|reserved_special_token_8|>",
134
  "lstrip": false,
135
  "normalized": false,
136
  "rstrip": false,
137
  "single_word": false,
138
+ "special": true
139
  },
140
  "128017": {
141
+ "content": "<|reserved_special_token_9|>",
142
  "lstrip": false,
143
  "normalized": false,
144
  "rstrip": false,
145
  "single_word": false,
146
+ "special": true
147
  },
148
  "128018": {
149
+ "content": "<|reserved_special_token_10|>",
150
  "lstrip": false,
151
  "normalized": false,
152
  "rstrip": false,
153
  "single_word": false,
154
+ "special": true
155
  },
156
  "128019": {
157
+ "content": "<|reserved_special_token_11|>",
158
  "lstrip": false,
159
  "normalized": false,
160
  "rstrip": false,
161
  "single_word": false,
162
+ "special": true
163
  },
164
  "128020": {
165
+ "content": "<|reserved_special_token_12|>",
166
  "lstrip": false,
167
  "normalized": false,
168
  "rstrip": false,
169
  "single_word": false,
170
+ "special": true
171
  },
172
  "128021": {
173
+ "content": "<|reserved_special_token_13|>",
174
  "lstrip": false,
175
  "normalized": false,
176
  "rstrip": false,
177
  "single_word": false,
178
+ "special": true
179
  },
180
  "128022": {
181
+ "content": "<|reserved_special_token_14|>",
182
  "lstrip": false,
183
  "normalized": false,
184
  "rstrip": false,
185
  "single_word": false,
186
+ "special": true
187
  },
188
  "128023": {
189
+ "content": "<|reserved_special_token_15|>",
190
  "lstrip": false,
191
  "normalized": false,
192
  "rstrip": false,
193
  "single_word": false,
194
+ "special": true
195
  },
196
  "128024": {
197
+ "content": "<|reserved_special_token_16|>",
198
  "lstrip": false,
199
  "normalized": false,
200
  "rstrip": false,
201
  "single_word": false,
202
+ "special": true
203
  },
204
  "128025": {
205
+ "content": "<|reserved_special_token_17|>",
206
  "lstrip": false,
207
  "normalized": false,
208
  "rstrip": false,
209
  "single_word": false,
210
+ "special": true
211
  },
212
  "128026": {
213
+ "content": "<|reserved_special_token_18|>",
214
  "lstrip": false,
215
  "normalized": false,
216
  "rstrip": false,
217
  "single_word": false,
218
+ "special": true
219
  },
220
  "128027": {
221
+ "content": "<|reserved_special_token_19|>",
222
  "lstrip": false,
223
  "normalized": false,
224
  "rstrip": false,
225
  "single_word": false,
226
+ "special": true
227
  },
228
  "128028": {
229
+ "content": "<|reserved_special_token_20|>",
230
  "lstrip": false,
231
  "normalized": false,
232
  "rstrip": false,
233
  "single_word": false,
234
+ "special": true
235
  },
236
  "128029": {
237
+ "content": "<|reserved_special_token_21|>",
238
  "lstrip": false,
239
  "normalized": false,
240
  "rstrip": false,
241
  "single_word": false,
242
+ "special": true
243
  },
244
  "128030": {
245
+ "content": "<|reserved_special_token_22|>",
246
  "lstrip": false,
247
  "normalized": false,
248
  "rstrip": false,
249
  "single_word": false,
250
+ "special": true
251
  },
252
  "128031": {
253
+ "content": "<|reserved_special_token_23|>",
254
  "lstrip": false,
255
  "normalized": false,
256
  "rstrip": false,
257
  "single_word": false,
258
+ "special": true
259
  },
260
  "128032": {
261
+ "content": "<|reserved_special_token_24|>",
262
  "lstrip": false,
263
  "normalized": false,
264
  "rstrip": false,
265
  "single_word": false,
266
+ "special": true
267
  },
268
  "128033": {
269
+ "content": "<|reserved_special_token_25|>",
270
  "lstrip": false,
271
  "normalized": false,
272
  "rstrip": false,
273
  "single_word": false,
274
+ "special": true
275
  },
276
  "128034": {
277
+ "content": "<|reserved_special_token_26|>",
278
  "lstrip": false,
279
  "normalized": false,
280
  "rstrip": false,
281
  "single_word": false,
282
+ "special": true
283
  },
284
  "128035": {
285
+ "content": "<|reserved_special_token_27|>",
286
  "lstrip": false,
287
  "normalized": false,
288
  "rstrip": false,
289
  "single_word": false,
290
+ "special": true
291
  },
292
  "128036": {
293
+ "content": "<|reserved_special_token_28|>",
294
  "lstrip": false,
295
  "normalized": false,
296
  "rstrip": false,
297
  "single_word": false,
298
+ "special": true
299
  },
300
  "128037": {
301
+ "content": "<|reserved_special_token_29|>",
302
  "lstrip": false,
303
  "normalized": false,
304
  "rstrip": false,
305
  "single_word": false,
306
+ "special": true
307
  },
308
  "128038": {
309
+ "content": "<|reserved_special_token_30|>",
310
  "lstrip": false,
311
  "normalized": false,
312
  "rstrip": false,
313
  "single_word": false,
314
+ "special": true
315
  },
316
  "128039": {
317
+ "content": "<|reserved_special_token_31|>",
318
  "lstrip": false,
319
  "normalized": false,
320
  "rstrip": false,
 
322
  "special": true
323
  },
324
  "128040": {
325
+ "content": "<|reserved_special_token_32|>",
326
  "lstrip": false,
327
  "normalized": false,
328
  "rstrip": false,
329
  "single_word": false,
330
+ "special": true
331
  },
332
  "128041": {
333
  "content": "<|reserved_special_token_33|>",