Config File Edited

config.json

@@ -11,23 +11,20 @@
       128009
     ],
     "hidden_act": "silu",
-    "hidden_size": 4096,
+    "hidden_size": 768,
     "initializer_range": 0.02,
     "intermediate_size": 14336,
     "max_position_embeddings": 131072,
     "mlp_bias": false,
     "model_type": "llama",
-    "num_attention_heads": 32,
+    "num_attention_heads": 12,
     "num_hidden_layers": 32,
     "num_key_value_heads": 8,
     "pretraining_tp": 1,
     "rms_norm_eps": 1e-05,
     "rope_scaling": {
       "factor": 8.0,
-      "low_freq_factor": 1.0,
-      "high_freq_factor": 4.0,
-      "original_max_position_embeddings": 8192,
-      "rope_type": "llama3"
+      "type": "linear"
     },
     "rope_theta": 500000.0,
     "tie_word_embeddings": false,