apinge commited on
Commit
7195ec8
·
verified ·
1 Parent(s): 955b884

Upload folder using huggingface_hub

Browse files
Files changed (4) hide show
  1. model.bin +3 -0
  2. model.xml +0 -0
  3. openvino_detokenizer.bin +3 -0
  4. openvino_detokenizer.xml +200 -0
model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:a4a67c7cf15bcff6fa4bc962f32c8b334bff4730dc161da47823023c8697bba4
3
+ size 468000828
model.xml ADDED
The diff for this file is too large to render. See raw diff
 
openvino_detokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:48a0af897d1991ff484190c0c4122783ba7dd1a71185555c6e07a55ad4ee3b50
3
+ size 379811
openvino_detokenizer.xml ADDED
@@ -0,0 +1,200 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0"?>
2
+ <net name="detokenizer" version="11">
3
+ <layers>
4
+ <layer id="0" name="Parameter_20" type="Parameter" version="opset1">
5
+ <data shape="?,?" element_type="i64" />
6
+ <output>
7
+ <port id="0" precision="I64" names="Parameter_20">
8
+ <dim>-1</dim>
9
+ <dim>-1</dim>
10
+ </port>
11
+ </output>
12
+ </layer>
13
+ <layer id="1" name="Constant_2" type="Const" version="opset1">
14
+ <data element_type="u8" shape="379758" offset="0" size="379758" />
15
+ <output>
16
+ <port id="0" precision="U8">
17
+ <dim>379758</dim>
18
+ </port>
19
+ </output>
20
+ </layer>
21
+ <layer id="2" name="Convert_36" type="Convert" version="opset1">
22
+ <data destination_type="i32" />
23
+ <input>
24
+ <port id="0" precision="I64">
25
+ <dim>-1</dim>
26
+ <dim>-1</dim>
27
+ </port>
28
+ </input>
29
+ <output>
30
+ <port id="1" precision="I32">
31
+ <dim>-1</dim>
32
+ <dim>-1</dim>
33
+ </port>
34
+ </output>
35
+ </layer>
36
+ <layer id="3" name="SentencepieceDetokenizer_21" type="SentencepieceDetokenizer" version="extension">
37
+ <input>
38
+ <port id="0" precision="U8">
39
+ <dim>379758</dim>
40
+ </port>
41
+ <port id="1" precision="I32">
42
+ <dim>-1</dim>
43
+ <dim>-1</dim>
44
+ </port>
45
+ </input>
46
+ <output>
47
+ <port id="2" precision="I32">
48
+ <dim>-1</dim>
49
+ </port>
50
+ <port id="3" precision="I32">
51
+ <dim>-1</dim>
52
+ </port>
53
+ <port id="4" precision="U8">
54
+ <dim>-1</dim>
55
+ </port>
56
+ </output>
57
+ </layer>
58
+ <layer id="4" name="Constant_23" type="Const" version="opset1">
59
+ <data element_type="u8" shape="51" offset="379758" size="51" />
60
+ <output>
61
+ <port id="0" precision="U8">
62
+ <dim>51</dim>
63
+ </port>
64
+ </output>
65
+ </layer>
66
+ <layer id="5" name="Constant_25" type="Const" version="opset1">
67
+ <data element_type="u8" shape="2" offset="379809" size="2" />
68
+ <output>
69
+ <port id="0" precision="U8">
70
+ <dim>2</dim>
71
+ </port>
72
+ </output>
73
+ </layer>
74
+ <layer id="6" name="RegexNormalization_26" type="RegexNormalization" version="extension">
75
+ <data global_replace="true" />
76
+ <input>
77
+ <port id="0" precision="I32">
78
+ <dim>-1</dim>
79
+ </port>
80
+ <port id="1" precision="I32">
81
+ <dim>-1</dim>
82
+ </port>
83
+ <port id="2" precision="U8">
84
+ <dim>-1</dim>
85
+ </port>
86
+ <port id="3" precision="U8">
87
+ <dim>51</dim>
88
+ </port>
89
+ <port id="4" precision="U8">
90
+ <dim>2</dim>
91
+ </port>
92
+ </input>
93
+ <output>
94
+ <port id="5" precision="I32">
95
+ <dim>-1</dim>
96
+ </port>
97
+ <port id="6" precision="I32">
98
+ <dim>-1</dim>
99
+ </port>
100
+ <port id="7" precision="U8">
101
+ <dim>-1</dim>
102
+ </port>
103
+ </output>
104
+ </layer>
105
+ <layer id="7" name="UTF8Validate_27" type="UTF8Validate" version="extension">
106
+ <data replace_mode="true" />
107
+ <input>
108
+ <port id="0" precision="I32">
109
+ <dim>-1</dim>
110
+ </port>
111
+ <port id="1" precision="I32">
112
+ <dim>-1</dim>
113
+ </port>
114
+ <port id="2" precision="U8">
115
+ <dim>-1</dim>
116
+ </port>
117
+ </input>
118
+ <output>
119
+ <port id="3" precision="I32">
120
+ <dim>-1</dim>
121
+ </port>
122
+ <port id="4" precision="I32">
123
+ <dim>-1</dim>
124
+ </port>
125
+ <port id="5" precision="U8">
126
+ <dim>-1</dim>
127
+ </port>
128
+ </output>
129
+ </layer>
130
+ <layer id="8" name="StringTensorPack_28" type="StringTensorPack" version="extension">
131
+ <data mode="begins_ends" />
132
+ <input>
133
+ <port id="0" precision="I32">
134
+ <dim>-1</dim>
135
+ </port>
136
+ <port id="1" precision="I32">
137
+ <dim>-1</dim>
138
+ </port>
139
+ <port id="2" precision="U8">
140
+ <dim>-1</dim>
141
+ </port>
142
+ </input>
143
+ <output>
144
+ <port id="3" precision="STRING" names="string_output">
145
+ <dim>-1</dim>
146
+ </port>
147
+ </output>
148
+ </layer>
149
+ <layer id="9" name="Result_29" type="Result" version="opset1">
150
+ <input>
151
+ <port id="0" precision="STRING">
152
+ <dim>-1</dim>
153
+ </port>
154
+ </input>
155
+ </layer>
156
+ </layers>
157
+ <edges>
158
+ <edge from-layer="0" from-port="0" to-layer="2" to-port="0" />
159
+ <edge from-layer="1" from-port="0" to-layer="3" to-port="0" />
160
+ <edge from-layer="2" from-port="1" to-layer="3" to-port="1" />
161
+ <edge from-layer="3" from-port="2" to-layer="6" to-port="0" />
162
+ <edge from-layer="3" from-port="3" to-layer="6" to-port="1" />
163
+ <edge from-layer="3" from-port="4" to-layer="6" to-port="2" />
164
+ <edge from-layer="4" from-port="0" to-layer="6" to-port="3" />
165
+ <edge from-layer="5" from-port="0" to-layer="6" to-port="4" />
166
+ <edge from-layer="6" from-port="5" to-layer="7" to-port="0" />
167
+ <edge from-layer="6" from-port="6" to-layer="7" to-port="1" />
168
+ <edge from-layer="6" from-port="7" to-layer="7" to-port="2" />
169
+ <edge from-layer="7" from-port="3" to-layer="8" to-port="0" />
170
+ <edge from-layer="7" from-port="4" to-layer="8" to-port="1" />
171
+ <edge from-layer="7" from-port="5" to-layer="8" to-port="2" />
172
+ <edge from-layer="8" from-port="3" to-layer="9" to-port="0" />
173
+ </edges>
174
+ <rt_info>
175
+ <add_attention_mask value="True" />
176
+ <add_prefix_space value="True" />
177
+ <add_special_tokens value="True" />
178
+ <chat_template value="{% for message in messages %}{{'&lt;|im_start|>' + message['role'] + '&#10;' + message['content'] + '&lt;|im_end|>' + '&#10;'}}{% endfor %}{% if add_generation_prompt %}{{ '&lt;|im_start|>assistant&#10;' }}{% endif %}" />
179
+ <clean_up_tokenization_spaces value="True" />
180
+ <detokenizer_input_type value="i64" />
181
+ <eos_token_id value="2" />
182
+ <handle_special_tokens_with_re value="False" />
183
+ <number_of_inputs value="1" />
184
+ <openvino_tokenizers_version value="2024.6.0.0" />
185
+ <openvino_version value="2024.6.0" />
186
+ <original_tokenizer_class value="&lt;class 'transformers.models.t5.tokenization_t5_fast.T5TokenizerFast'>" />
187
+ <pad_token_id value="25155" />
188
+ <sentencepiece_version value="0.2.0" />
189
+ <skip_special_tokens value="True" />
190
+ <streaming_detokenizer value="False" />
191
+ <tiktoken_version value="0.8.0" />
192
+ <tokenizer_output_type value="i64" />
193
+ <tokenizers_version value="0.19.1" />
194
+ <transformers_version value="4.40.1" />
195
+ <use_max_padding value="False" />
196
+ <use_sentencepiece_backend value="False" />
197
+ <utf8_replace_mode value="replace" />
198
+ <with_detokenizer value="True" />
199
+ </rt_info>
200
+ </net>