Training in progress, step 500

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "meta-llama/Llama-3.1-8B",
+  "_name_or_path": "meta-llama/Llama-3.2-3B",
   "architectures": [
     "LlamaForQuestionAnswering"
   ],
@@ -9,26 +9,26 @@
   "eos_token_id": 128001,
   "head_dim": 128,
   "hidden_act": "silu",
-  "hidden_size": 4096,
+  "hidden_size": 3072,
   "initializer_range": 0.02,
-  "intermediate_size": 14336,
+  "intermediate_size": 8192,
   "max_position_embeddings": 131072,
   "mlp_bias": false,
   "model_type": "llama",
-  "num_attention_heads": 32,
-  "num_hidden_layers": 32,
+  "num_attention_heads": 24,
+  "num_hidden_layers": 28,
   "num_key_value_heads": 8,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
-    "factor": 8.0,
+    "factor": 32.0,
     "high_freq_factor": 4.0,
     "low_freq_factor": 1.0,
     "original_max_position_embeddings": 8192,
     "rope_type": "llama3"
   },
   "rope_theta": 500000.0,
-  "tie_word_embeddings": false,
+  "tie_word_embeddings": true,
   "torch_dtype": "float32",
   "transformers_version": "4.47.1",
   "use_cache": true,

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:375f8b6e774f8e79e194e4d7d7a51a1c4605e21b769f5dda8d4c95ada7b8b7ea
+oid sha256:0cf2b1fc83d69410c791e6026f17d1391aa398b5920a91e2a62c9ab87714de62
 size 4998780112

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7d866cb8b30104c35f6e1d898fc5d831ee6d69b5f6b126dc288a8505ed6afe89
+oid sha256:1c745eefb3f841fb4d5c056bcdb6702575c535b223988f4a426d04bcb52d66a0
 size 4932809624

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:835a05ecb2030cd2e9e53688f8d52961fe37450fa1a3f9f8e586f8f255afcfb0
+oid sha256:6cc3ef6f6f88922b4d0af7a02bf791dd65da96b0599d396e5112f5c4cef120ae
 size 2919477304

model.safetensors.index.json

@@ -1,299 +1,263 @@
 {
   "metadata": {
-    "total_size": 30019747848
+    "total_size": 12851036168
   },
   "weight_map": {
-    "qa_outputs.bias": "model-00007-of-00007.safetensors",
-    "qa_outputs.weight": "model-00007-of-00007.safetensors",
-    "transformer.embed_tokens.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.input_layernorm.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.0.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.input_layernorm.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.1.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.10.input_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.10.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.input_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.11.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.input_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.12.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.input_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.13.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.14.input_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.14.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.14.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.14.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.14.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.14.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.14.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.14.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.14.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.15.input_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.15.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.input_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.16.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.input_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.17.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.input_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.18.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.input_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.mlp.down_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.mlp.gate_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.mlp.up_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.post_attention_layernorm.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.19.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.2.input_layernorm.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.mlp.down_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.mlp.gate_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.mlp.up_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.post_attention_layernorm.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.2.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.20.input_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.20.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.20.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.20.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.20.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.20.self_attn.k_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.20.self_attn.o_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.20.self_attn.q_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.20.self_attn.v_proj.weight": "model-00004-of-00007.safetensors",
-    "transformer.layers.21.input_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.21.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.input_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.22.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.input_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.23.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.input_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.mlp.down_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.post_attention_layernorm.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.24.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.25.input_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.25.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.25.mlp.gate_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.25.mlp.up_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.25.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.25.self_attn.k_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.25.self_attn.o_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.25.self_attn.q_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.25.self_attn.v_proj.weight": "model-00005-of-00007.safetensors",
-    "transformer.layers.26.input_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.26.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.input_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.27.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.input_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.28.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.input_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.29.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.3.input_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.3.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.3.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.3.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.3.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.3.self_attn.k_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.3.self_attn.o_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.3.self_attn.q_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.3.self_attn.v_proj.weight": "model-00001-of-00007.safetensors",
-    "transformer.layers.30.input_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.mlp.down_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.mlp.up_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.post_attention_layernorm.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.30.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.31.input_layernorm.weight": "model-00007-of-00007.safetensors",
-    "transformer.layers.31.mlp.down_proj.weight": "model-00007-of-00007.safetensors",
-    "transformer.layers.31.mlp.gate_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.31.mlp.up_proj.weight": "model-00007-of-00007.safetensors",
-    "transformer.layers.31.post_attention_layernorm.weight": "model-00007-of-00007.safetensors",
-    "transformer.layers.31.self_attn.k_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.31.self_attn.o_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.31.self_attn.q_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.31.self_attn.v_proj.weight": "model-00006-of-00007.safetensors",
-    "transformer.layers.4.input_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.4.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.input_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.5.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.input_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.6.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.input_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.mlp.down_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.post_attention_layernorm.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.7.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.8.input_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.8.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.8.mlp.gate_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.8.mlp.up_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.8.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.8.self_attn.k_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.8.self_attn.o_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.8.self_attn.q_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.8.self_attn.v_proj.weight": "model-00002-of-00007.safetensors",
-    "transformer.layers.9.input_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.mlp.down_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.mlp.gate_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.mlp.up_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.post_attention_layernorm.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.self_attn.k_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.self_attn.o_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.self_attn.q_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.layers.9.self_attn.v_proj.weight": "model-00003-of-00007.safetensors",
-    "transformer.norm.weight": "model-00007-of-00007.safetensors"
+    "qa_outputs.bias": "model-00003-of-00003.safetensors",
+    "qa_outputs.weight": "model-00003-of-00003.safetensors",
+    "transformer.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.10.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.10.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.11.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.20.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.20.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.20.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.21.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.21.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.22.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.23.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "transformer.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.8.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.8.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.8.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.8.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "transformer.layers.9.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.layers.9.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "transformer.norm.weight": "model-00003-of-00003.safetensors"
   }
 }

runs/Jan08_21-26-21_xgpi3/events.out.tfevents.1736342848.xgpi3.182970.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ac420a31a835d27cbf86246c708cb9a7bfdb76f5b96d0ace768f38f1a81bd62b
+size 5516

tokenizer.json

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e8485b8900f2ef3a891091dbe5ef0edc56f6922c0a23c7dfbb4baf64854f7de4
-size 17210102
+oid sha256:1f071ee1cdf961211cd4f474612169091a389e3c79a72d01ec50399be95369a6
+size 17210372

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f31b188fa6f6e23260836fe03c8dd965c06b1aa2cac5291a69866750dd0e3907
+oid sha256:b3c3590ac7a57ae4dfa46327f4c5fb4b5926ac23116fb5b0dcc793309ec9489f
 size 5368