yzhangcs commited on
Commit
e29b06c
·
verified ·
1 Parent(s): 35c70c1

Upload TransformerForCausalLM

Browse files
README.md CHANGED
@@ -8,4 +8,4 @@ license: mit
8
  datasets:
9
  - cerebras/SlimPajama-627B
10
  library_name: fla
11
- ---
 
8
  datasets:
9
  - cerebras/SlimPajama-627B
10
  library_name: fla
11
+ ---
config.json CHANGED
@@ -5,9 +5,11 @@
5
  ],
6
  "attention_bias": false,
7
  "bos_token_id": 1,
 
8
  "eos_token_id": 2,
9
  "fuse_cross_entropy": true,
10
  "fuse_norm": true,
 
11
  "hidden_act": "swish",
12
  "hidden_ratio": 4,
13
  "hidden_size": 2560,
@@ -15,12 +17,16 @@
15
  "intermediate_size": null,
16
  "max_position_embeddings": 2048,
17
  "model_type": "transformer",
 
18
  "num_heads": 20,
19
  "num_hidden_layers": 32,
 
20
  "rms_norm_eps": 1e-06,
 
21
  "tie_word_embeddings": false,
22
  "torch_dtype": "bfloat16",
23
- "transformers_version": "4.40.0.dev0",
24
  "use_cache": true,
25
- "vocab_size": 32000
 
26
  }
 
5
  ],
6
  "attention_bias": false,
7
  "bos_token_id": 1,
8
+ "elementwise_affine": true,
9
  "eos_token_id": 2,
10
  "fuse_cross_entropy": true,
11
  "fuse_norm": true,
12
+ "fuse_swiglu": true,
13
  "hidden_act": "swish",
14
  "hidden_ratio": 4,
15
  "hidden_size": 2560,
 
17
  "intermediate_size": null,
18
  "max_position_embeddings": 2048,
19
  "model_type": "transformer",
20
+ "norm_eps": 1e-06,
21
  "num_heads": 20,
22
  "num_hidden_layers": 32,
23
+ "num_kv_heads": null,
24
  "rms_norm_eps": 1e-06,
25
+ "rope_theta": 10000.0,
26
  "tie_word_embeddings": false,
27
  "torch_dtype": "bfloat16",
28
+ "transformers_version": "4.48.2",
29
  "use_cache": true,
30
+ "vocab_size": 32000,
31
+ "window_size": null
32
  }
generation_config.json CHANGED
@@ -2,5 +2,5 @@
2
  "_from_model_config": true,
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
- "transformers_version": "4.40.0.dev0"
6
  }
 
2
  "_from_model_config": true,
3
  "bos_token_id": 1,
4
  "eos_token_id": 2,
5
+ "transformers_version": "4.48.2"
6
  }
model-00001-of-00002.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:42d813182eba43748db525a7a505ff1a649a9e478e83c68a5370a65cad7ae62a
3
- size 4974527296
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:19c08e94c9d4c4a985d07b57823fa7aa589becb294af61a454b69f18f35ce360
3
+ size 4974530632
model-00002-of-00002.safetensors CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:44b900ebf5fb40a51fcec941fcc7badad77b9cf8be2d3ef056f7adf30b54ded3
3
- size 428622120
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1960431ae037a4a71d147ca442c04d3810bbe9f2f2f564b286616c0b77235c72
3
+ size 428622344
model.safetensors.index.json CHANGED
@@ -12,6 +12,7 @@
12
  "model.layers.0.attn_norm.weight": "model-00001-of-00002.safetensors",
13
  "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
14
  "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
15
  "model.layers.0.mlp_norm.weight": "model-00001-of-00002.safetensors",
16
  "model.layers.1.attn.k_proj.weight": "model-00001-of-00002.safetensors",
17
  "model.layers.1.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -20,6 +21,7 @@
20
  "model.layers.1.attn_norm.weight": "model-00001-of-00002.safetensors",
21
  "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
22
  "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
23
  "model.layers.1.mlp_norm.weight": "model-00001-of-00002.safetensors",
24
  "model.layers.10.attn.k_proj.weight": "model-00001-of-00002.safetensors",
25
  "model.layers.10.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -28,6 +30,7 @@
28
  "model.layers.10.attn_norm.weight": "model-00001-of-00002.safetensors",
29
  "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
30
  "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
31
  "model.layers.10.mlp_norm.weight": "model-00001-of-00002.safetensors",
32
  "model.layers.11.attn.k_proj.weight": "model-00001-of-00002.safetensors",
33
  "model.layers.11.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -36,6 +39,7 @@
36
  "model.layers.11.attn_norm.weight": "model-00001-of-00002.safetensors",
37
  "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
38
  "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
39
  "model.layers.11.mlp_norm.weight": "model-00001-of-00002.safetensors",
40
  "model.layers.12.attn.k_proj.weight": "model-00001-of-00002.safetensors",
41
  "model.layers.12.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -44,6 +48,7 @@
44
  "model.layers.12.attn_norm.weight": "model-00001-of-00002.safetensors",
45
  "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
46
  "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
47
  "model.layers.12.mlp_norm.weight": "model-00001-of-00002.safetensors",
48
  "model.layers.13.attn.k_proj.weight": "model-00001-of-00002.safetensors",
49
  "model.layers.13.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -52,6 +57,7 @@
52
  "model.layers.13.attn_norm.weight": "model-00001-of-00002.safetensors",
53
  "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
54
  "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
55
  "model.layers.13.mlp_norm.weight": "model-00001-of-00002.safetensors",
56
  "model.layers.14.attn.k_proj.weight": "model-00001-of-00002.safetensors",
57
  "model.layers.14.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -60,6 +66,7 @@
60
  "model.layers.14.attn_norm.weight": "model-00001-of-00002.safetensors",
61
  "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
62
  "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
63
  "model.layers.14.mlp_norm.weight": "model-00001-of-00002.safetensors",
64
  "model.layers.15.attn.k_proj.weight": "model-00001-of-00002.safetensors",
65
  "model.layers.15.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -68,6 +75,7 @@
68
  "model.layers.15.attn_norm.weight": "model-00001-of-00002.safetensors",
69
  "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
70
  "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
71
  "model.layers.15.mlp_norm.weight": "model-00001-of-00002.safetensors",
72
  "model.layers.16.attn.k_proj.weight": "model-00001-of-00002.safetensors",
73
  "model.layers.16.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -76,6 +84,7 @@
76
  "model.layers.16.attn_norm.weight": "model-00001-of-00002.safetensors",
77
  "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
78
  "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
79
  "model.layers.16.mlp_norm.weight": "model-00001-of-00002.safetensors",
80
  "model.layers.17.attn.k_proj.weight": "model-00001-of-00002.safetensors",
81
  "model.layers.17.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -84,6 +93,7 @@
84
  "model.layers.17.attn_norm.weight": "model-00001-of-00002.safetensors",
85
  "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
86
  "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
87
  "model.layers.17.mlp_norm.weight": "model-00001-of-00002.safetensors",
88
  "model.layers.18.attn.k_proj.weight": "model-00001-of-00002.safetensors",
89
  "model.layers.18.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -92,6 +102,7 @@
92
  "model.layers.18.attn_norm.weight": "model-00001-of-00002.safetensors",
93
  "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
94
  "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
95
  "model.layers.18.mlp_norm.weight": "model-00001-of-00002.safetensors",
96
  "model.layers.19.attn.k_proj.weight": "model-00001-of-00002.safetensors",
97
  "model.layers.19.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -100,6 +111,7 @@
100
  "model.layers.19.attn_norm.weight": "model-00001-of-00002.safetensors",
101
  "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
102
  "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
103
  "model.layers.19.mlp_norm.weight": "model-00001-of-00002.safetensors",
104
  "model.layers.2.attn.k_proj.weight": "model-00001-of-00002.safetensors",
105
  "model.layers.2.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -108,6 +120,7 @@
108
  "model.layers.2.attn_norm.weight": "model-00001-of-00002.safetensors",
109
  "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
110
  "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
111
  "model.layers.2.mlp_norm.weight": "model-00001-of-00002.safetensors",
112
  "model.layers.20.attn.k_proj.weight": "model-00001-of-00002.safetensors",
113
  "model.layers.20.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -116,6 +129,7 @@
116
  "model.layers.20.attn_norm.weight": "model-00001-of-00002.safetensors",
117
  "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
118
  "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
119
  "model.layers.20.mlp_norm.weight": "model-00001-of-00002.safetensors",
120
  "model.layers.21.attn.k_proj.weight": "model-00001-of-00002.safetensors",
121
  "model.layers.21.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -124,6 +138,7 @@
124
  "model.layers.21.attn_norm.weight": "model-00001-of-00002.safetensors",
125
  "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
126
  "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
127
  "model.layers.21.mlp_norm.weight": "model-00001-of-00002.safetensors",
128
  "model.layers.22.attn.k_proj.weight": "model-00001-of-00002.safetensors",
129
  "model.layers.22.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -132,6 +147,7 @@
132
  "model.layers.22.attn_norm.weight": "model-00001-of-00002.safetensors",
133
  "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
134
  "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
135
  "model.layers.22.mlp_norm.weight": "model-00001-of-00002.safetensors",
136
  "model.layers.23.attn.k_proj.weight": "model-00001-of-00002.safetensors",
137
  "model.layers.23.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -140,6 +156,7 @@
140
  "model.layers.23.attn_norm.weight": "model-00001-of-00002.safetensors",
141
  "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
142
  "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
143
  "model.layers.23.mlp_norm.weight": "model-00001-of-00002.safetensors",
144
  "model.layers.24.attn.k_proj.weight": "model-00001-of-00002.safetensors",
145
  "model.layers.24.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -148,6 +165,7 @@
148
  "model.layers.24.attn_norm.weight": "model-00001-of-00002.safetensors",
149
  "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
150
  "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
151
  "model.layers.24.mlp_norm.weight": "model-00001-of-00002.safetensors",
152
  "model.layers.25.attn.k_proj.weight": "model-00001-of-00002.safetensors",
153
  "model.layers.25.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -156,6 +174,7 @@
156
  "model.layers.25.attn_norm.weight": "model-00001-of-00002.safetensors",
157
  "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
158
  "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
159
  "model.layers.25.mlp_norm.weight": "model-00001-of-00002.safetensors",
160
  "model.layers.26.attn.k_proj.weight": "model-00001-of-00002.safetensors",
161
  "model.layers.26.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -164,6 +183,7 @@
164
  "model.layers.26.attn_norm.weight": "model-00001-of-00002.safetensors",
165
  "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
166
  "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
167
  "model.layers.26.mlp_norm.weight": "model-00001-of-00002.safetensors",
168
  "model.layers.27.attn.k_proj.weight": "model-00001-of-00002.safetensors",
169
  "model.layers.27.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -172,6 +192,7 @@
172
  "model.layers.27.attn_norm.weight": "model-00001-of-00002.safetensors",
173
  "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
174
  "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
175
  "model.layers.27.mlp_norm.weight": "model-00001-of-00002.safetensors",
176
  "model.layers.28.attn.k_proj.weight": "model-00001-of-00002.safetensors",
177
  "model.layers.28.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -180,6 +201,7 @@
180
  "model.layers.28.attn_norm.weight": "model-00001-of-00002.safetensors",
181
  "model.layers.28.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
182
  "model.layers.28.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
183
  "model.layers.28.mlp_norm.weight": "model-00001-of-00002.safetensors",
184
  "model.layers.29.attn.k_proj.weight": "model-00001-of-00002.safetensors",
185
  "model.layers.29.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -188,6 +210,7 @@
188
  "model.layers.29.attn_norm.weight": "model-00001-of-00002.safetensors",
189
  "model.layers.29.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
190
  "model.layers.29.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
191
  "model.layers.29.mlp_norm.weight": "model-00001-of-00002.safetensors",
192
  "model.layers.3.attn.k_proj.weight": "model-00001-of-00002.safetensors",
193
  "model.layers.3.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -196,6 +219,7 @@
196
  "model.layers.3.attn_norm.weight": "model-00001-of-00002.safetensors",
197
  "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
198
  "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
199
  "model.layers.3.mlp_norm.weight": "model-00001-of-00002.safetensors",
200
  "model.layers.30.attn.k_proj.weight": "model-00001-of-00002.safetensors",
201
  "model.layers.30.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -204,6 +228,7 @@
204
  "model.layers.30.attn_norm.weight": "model-00001-of-00002.safetensors",
205
  "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
206
  "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
 
207
  "model.layers.30.mlp_norm.weight": "model-00001-of-00002.safetensors",
208
  "model.layers.31.attn.k_proj.weight": "model-00002-of-00002.safetensors",
209
  "model.layers.31.attn.o_proj.weight": "model-00002-of-00002.safetensors",
@@ -212,6 +237,7 @@
212
  "model.layers.31.attn_norm.weight": "model-00002-of-00002.safetensors",
213
  "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
214
  "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
 
215
  "model.layers.31.mlp_norm.weight": "model-00002-of-00002.safetensors",
216
  "model.layers.4.attn.k_proj.weight": "model-00001-of-00002.safetensors",
217
  "model.layers.4.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -220,6 +246,7 @@
220
  "model.layers.4.attn_norm.weight": "model-00001-of-00002.safetensors",
221
  "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
222
  "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
223
  "model.layers.4.mlp_norm.weight": "model-00001-of-00002.safetensors",
224
  "model.layers.5.attn.k_proj.weight": "model-00001-of-00002.safetensors",
225
  "model.layers.5.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -228,6 +255,7 @@
228
  "model.layers.5.attn_norm.weight": "model-00001-of-00002.safetensors",
229
  "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
230
  "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
231
  "model.layers.5.mlp_norm.weight": "model-00001-of-00002.safetensors",
232
  "model.layers.6.attn.k_proj.weight": "model-00001-of-00002.safetensors",
233
  "model.layers.6.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -236,6 +264,7 @@
236
  "model.layers.6.attn_norm.weight": "model-00001-of-00002.safetensors",
237
  "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
238
  "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
239
  "model.layers.6.mlp_norm.weight": "model-00001-of-00002.safetensors",
240
  "model.layers.7.attn.k_proj.weight": "model-00001-of-00002.safetensors",
241
  "model.layers.7.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -244,6 +273,7 @@
244
  "model.layers.7.attn_norm.weight": "model-00001-of-00002.safetensors",
245
  "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
246
  "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
247
  "model.layers.7.mlp_norm.weight": "model-00001-of-00002.safetensors",
248
  "model.layers.8.attn.k_proj.weight": "model-00001-of-00002.safetensors",
249
  "model.layers.8.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -252,6 +282,7 @@
252
  "model.layers.8.attn_norm.weight": "model-00001-of-00002.safetensors",
253
  "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
254
  "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
255
  "model.layers.8.mlp_norm.weight": "model-00001-of-00002.safetensors",
256
  "model.layers.9.attn.k_proj.weight": "model-00001-of-00002.safetensors",
257
  "model.layers.9.attn.o_proj.weight": "model-00001-of-00002.safetensors",
@@ -260,6 +291,7 @@
260
  "model.layers.9.attn_norm.weight": "model-00001-of-00002.safetensors",
261
  "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
262
  "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
 
263
  "model.layers.9.mlp_norm.weight": "model-00001-of-00002.safetensors",
264
  "model.norm.weight": "model-00002-of-00002.safetensors"
265
  }
 
12
  "model.layers.0.attn_norm.weight": "model-00001-of-00002.safetensors",
13
  "model.layers.0.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
14
  "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
15
+ "model.layers.0.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
16
  "model.layers.0.mlp_norm.weight": "model-00001-of-00002.safetensors",
17
  "model.layers.1.attn.k_proj.weight": "model-00001-of-00002.safetensors",
18
  "model.layers.1.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
21
  "model.layers.1.attn_norm.weight": "model-00001-of-00002.safetensors",
22
  "model.layers.1.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
23
  "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
24
+ "model.layers.1.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
25
  "model.layers.1.mlp_norm.weight": "model-00001-of-00002.safetensors",
26
  "model.layers.10.attn.k_proj.weight": "model-00001-of-00002.safetensors",
27
  "model.layers.10.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
30
  "model.layers.10.attn_norm.weight": "model-00001-of-00002.safetensors",
31
  "model.layers.10.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
32
  "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
33
+ "model.layers.10.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
34
  "model.layers.10.mlp_norm.weight": "model-00001-of-00002.safetensors",
35
  "model.layers.11.attn.k_proj.weight": "model-00001-of-00002.safetensors",
36
  "model.layers.11.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
39
  "model.layers.11.attn_norm.weight": "model-00001-of-00002.safetensors",
40
  "model.layers.11.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
41
  "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
42
+ "model.layers.11.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
43
  "model.layers.11.mlp_norm.weight": "model-00001-of-00002.safetensors",
44
  "model.layers.12.attn.k_proj.weight": "model-00001-of-00002.safetensors",
45
  "model.layers.12.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
48
  "model.layers.12.attn_norm.weight": "model-00001-of-00002.safetensors",
49
  "model.layers.12.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
50
  "model.layers.12.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
51
+ "model.layers.12.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
52
  "model.layers.12.mlp_norm.weight": "model-00001-of-00002.safetensors",
53
  "model.layers.13.attn.k_proj.weight": "model-00001-of-00002.safetensors",
54
  "model.layers.13.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
57
  "model.layers.13.attn_norm.weight": "model-00001-of-00002.safetensors",
58
  "model.layers.13.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
59
  "model.layers.13.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
60
+ "model.layers.13.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
61
  "model.layers.13.mlp_norm.weight": "model-00001-of-00002.safetensors",
62
  "model.layers.14.attn.k_proj.weight": "model-00001-of-00002.safetensors",
63
  "model.layers.14.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
66
  "model.layers.14.attn_norm.weight": "model-00001-of-00002.safetensors",
67
  "model.layers.14.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
68
  "model.layers.14.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
69
+ "model.layers.14.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
70
  "model.layers.14.mlp_norm.weight": "model-00001-of-00002.safetensors",
71
  "model.layers.15.attn.k_proj.weight": "model-00001-of-00002.safetensors",
72
  "model.layers.15.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
75
  "model.layers.15.attn_norm.weight": "model-00001-of-00002.safetensors",
76
  "model.layers.15.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
77
  "model.layers.15.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
78
+ "model.layers.15.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
79
  "model.layers.15.mlp_norm.weight": "model-00001-of-00002.safetensors",
80
  "model.layers.16.attn.k_proj.weight": "model-00001-of-00002.safetensors",
81
  "model.layers.16.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
84
  "model.layers.16.attn_norm.weight": "model-00001-of-00002.safetensors",
85
  "model.layers.16.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
86
  "model.layers.16.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
87
+ "model.layers.16.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
88
  "model.layers.16.mlp_norm.weight": "model-00001-of-00002.safetensors",
89
  "model.layers.17.attn.k_proj.weight": "model-00001-of-00002.safetensors",
90
  "model.layers.17.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
93
  "model.layers.17.attn_norm.weight": "model-00001-of-00002.safetensors",
94
  "model.layers.17.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
95
  "model.layers.17.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
96
+ "model.layers.17.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
97
  "model.layers.17.mlp_norm.weight": "model-00001-of-00002.safetensors",
98
  "model.layers.18.attn.k_proj.weight": "model-00001-of-00002.safetensors",
99
  "model.layers.18.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
102
  "model.layers.18.attn_norm.weight": "model-00001-of-00002.safetensors",
103
  "model.layers.18.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
104
  "model.layers.18.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
105
+ "model.layers.18.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
106
  "model.layers.18.mlp_norm.weight": "model-00001-of-00002.safetensors",
107
  "model.layers.19.attn.k_proj.weight": "model-00001-of-00002.safetensors",
108
  "model.layers.19.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
111
  "model.layers.19.attn_norm.weight": "model-00001-of-00002.safetensors",
112
  "model.layers.19.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
113
  "model.layers.19.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
114
+ "model.layers.19.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
115
  "model.layers.19.mlp_norm.weight": "model-00001-of-00002.safetensors",
116
  "model.layers.2.attn.k_proj.weight": "model-00001-of-00002.safetensors",
117
  "model.layers.2.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
120
  "model.layers.2.attn_norm.weight": "model-00001-of-00002.safetensors",
121
  "model.layers.2.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
122
  "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
123
+ "model.layers.2.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
124
  "model.layers.2.mlp_norm.weight": "model-00001-of-00002.safetensors",
125
  "model.layers.20.attn.k_proj.weight": "model-00001-of-00002.safetensors",
126
  "model.layers.20.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
129
  "model.layers.20.attn_norm.weight": "model-00001-of-00002.safetensors",
130
  "model.layers.20.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
131
  "model.layers.20.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
132
+ "model.layers.20.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
133
  "model.layers.20.mlp_norm.weight": "model-00001-of-00002.safetensors",
134
  "model.layers.21.attn.k_proj.weight": "model-00001-of-00002.safetensors",
135
  "model.layers.21.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
138
  "model.layers.21.attn_norm.weight": "model-00001-of-00002.safetensors",
139
  "model.layers.21.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
140
  "model.layers.21.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
141
+ "model.layers.21.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
142
  "model.layers.21.mlp_norm.weight": "model-00001-of-00002.safetensors",
143
  "model.layers.22.attn.k_proj.weight": "model-00001-of-00002.safetensors",
144
  "model.layers.22.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
147
  "model.layers.22.attn_norm.weight": "model-00001-of-00002.safetensors",
148
  "model.layers.22.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
149
  "model.layers.22.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
150
+ "model.layers.22.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
151
  "model.layers.22.mlp_norm.weight": "model-00001-of-00002.safetensors",
152
  "model.layers.23.attn.k_proj.weight": "model-00001-of-00002.safetensors",
153
  "model.layers.23.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
156
  "model.layers.23.attn_norm.weight": "model-00001-of-00002.safetensors",
157
  "model.layers.23.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
158
  "model.layers.23.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
159
+ "model.layers.23.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
160
  "model.layers.23.mlp_norm.weight": "model-00001-of-00002.safetensors",
161
  "model.layers.24.attn.k_proj.weight": "model-00001-of-00002.safetensors",
162
  "model.layers.24.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
165
  "model.layers.24.attn_norm.weight": "model-00001-of-00002.safetensors",
166
  "model.layers.24.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
167
  "model.layers.24.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
168
+ "model.layers.24.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
169
  "model.layers.24.mlp_norm.weight": "model-00001-of-00002.safetensors",
170
  "model.layers.25.attn.k_proj.weight": "model-00001-of-00002.safetensors",
171
  "model.layers.25.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
174
  "model.layers.25.attn_norm.weight": "model-00001-of-00002.safetensors",
175
  "model.layers.25.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
176
  "model.layers.25.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
177
+ "model.layers.25.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
178
  "model.layers.25.mlp_norm.weight": "model-00001-of-00002.safetensors",
179
  "model.layers.26.attn.k_proj.weight": "model-00001-of-00002.safetensors",
180
  "model.layers.26.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
183
  "model.layers.26.attn_norm.weight": "model-00001-of-00002.safetensors",
184
  "model.layers.26.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
185
  "model.layers.26.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
186
+ "model.layers.26.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
187
  "model.layers.26.mlp_norm.weight": "model-00001-of-00002.safetensors",
188
  "model.layers.27.attn.k_proj.weight": "model-00001-of-00002.safetensors",
189
  "model.layers.27.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
192
  "model.layers.27.attn_norm.weight": "model-00001-of-00002.safetensors",
193
  "model.layers.27.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
194
  "model.layers.27.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
195
+ "model.layers.27.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
196
  "model.layers.27.mlp_norm.weight": "model-00001-of-00002.safetensors",
197
  "model.layers.28.attn.k_proj.weight": "model-00001-of-00002.safetensors",
198
  "model.layers.28.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
201
  "model.layers.28.attn_norm.weight": "model-00001-of-00002.safetensors",
202
  "model.layers.28.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
203
  "model.layers.28.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
204
+ "model.layers.28.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
205
  "model.layers.28.mlp_norm.weight": "model-00001-of-00002.safetensors",
206
  "model.layers.29.attn.k_proj.weight": "model-00001-of-00002.safetensors",
207
  "model.layers.29.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
210
  "model.layers.29.attn_norm.weight": "model-00001-of-00002.safetensors",
211
  "model.layers.29.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
212
  "model.layers.29.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
213
+ "model.layers.29.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
214
  "model.layers.29.mlp_norm.weight": "model-00001-of-00002.safetensors",
215
  "model.layers.3.attn.k_proj.weight": "model-00001-of-00002.safetensors",
216
  "model.layers.3.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
219
  "model.layers.3.attn_norm.weight": "model-00001-of-00002.safetensors",
220
  "model.layers.3.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
221
  "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
222
+ "model.layers.3.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
223
  "model.layers.3.mlp_norm.weight": "model-00001-of-00002.safetensors",
224
  "model.layers.30.attn.k_proj.weight": "model-00001-of-00002.safetensors",
225
  "model.layers.30.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
228
  "model.layers.30.attn_norm.weight": "model-00001-of-00002.safetensors",
229
  "model.layers.30.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
230
  "model.layers.30.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
231
+ "model.layers.30.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
232
  "model.layers.30.mlp_norm.weight": "model-00001-of-00002.safetensors",
233
  "model.layers.31.attn.k_proj.weight": "model-00002-of-00002.safetensors",
234
  "model.layers.31.attn.o_proj.weight": "model-00002-of-00002.safetensors",
 
237
  "model.layers.31.attn_norm.weight": "model-00002-of-00002.safetensors",
238
  "model.layers.31.mlp.down_proj.weight": "model-00002-of-00002.safetensors",
239
  "model.layers.31.mlp.gate_proj.weight": "model-00002-of-00002.safetensors",
240
+ "model.layers.31.mlp.up_proj.weight": "model-00002-of-00002.safetensors",
241
  "model.layers.31.mlp_norm.weight": "model-00002-of-00002.safetensors",
242
  "model.layers.4.attn.k_proj.weight": "model-00001-of-00002.safetensors",
243
  "model.layers.4.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
246
  "model.layers.4.attn_norm.weight": "model-00001-of-00002.safetensors",
247
  "model.layers.4.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
248
  "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
249
+ "model.layers.4.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
250
  "model.layers.4.mlp_norm.weight": "model-00001-of-00002.safetensors",
251
  "model.layers.5.attn.k_proj.weight": "model-00001-of-00002.safetensors",
252
  "model.layers.5.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
255
  "model.layers.5.attn_norm.weight": "model-00001-of-00002.safetensors",
256
  "model.layers.5.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
257
  "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
258
+ "model.layers.5.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
259
  "model.layers.5.mlp_norm.weight": "model-00001-of-00002.safetensors",
260
  "model.layers.6.attn.k_proj.weight": "model-00001-of-00002.safetensors",
261
  "model.layers.6.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
264
  "model.layers.6.attn_norm.weight": "model-00001-of-00002.safetensors",
265
  "model.layers.6.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
266
  "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
267
+ "model.layers.6.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
268
  "model.layers.6.mlp_norm.weight": "model-00001-of-00002.safetensors",
269
  "model.layers.7.attn.k_proj.weight": "model-00001-of-00002.safetensors",
270
  "model.layers.7.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
273
  "model.layers.7.attn_norm.weight": "model-00001-of-00002.safetensors",
274
  "model.layers.7.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
275
  "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
276
+ "model.layers.7.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
277
  "model.layers.7.mlp_norm.weight": "model-00001-of-00002.safetensors",
278
  "model.layers.8.attn.k_proj.weight": "model-00001-of-00002.safetensors",
279
  "model.layers.8.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
282
  "model.layers.8.attn_norm.weight": "model-00001-of-00002.safetensors",
283
  "model.layers.8.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
284
  "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
285
+ "model.layers.8.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
286
  "model.layers.8.mlp_norm.weight": "model-00001-of-00002.safetensors",
287
  "model.layers.9.attn.k_proj.weight": "model-00001-of-00002.safetensors",
288
  "model.layers.9.attn.o_proj.weight": "model-00001-of-00002.safetensors",
 
291
  "model.layers.9.attn_norm.weight": "model-00001-of-00002.safetensors",
292
  "model.layers.9.mlp.down_proj.weight": "model-00001-of-00002.safetensors",
293
  "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00002.safetensors",
294
+ "model.layers.9.mlp.up_proj.weight": "model-00001-of-00002.safetensors",
295
  "model.layers.9.mlp_norm.weight": "model-00001-of-00002.safetensors",
296
  "model.norm.weight": "model-00002-of-00002.safetensors"
297
  }