Upload tiny random Llama-3.3 model (seed 42)

README.md

@@ -16,16 +16,16 @@ This is a tiny random version of the EleutherAI/pythia-14m model, created for te
 
 - **Base model**: EleutherAI/pythia-14m
 - **Seed**: 42
-- **Hidden size**: 64
-- **Number of layers**: 2
-- **Number of attention heads**: 2
-- **Vocabulary size**: 1000
-- **Max position embeddings**: 512
+- **Hidden size**: 128
+- **Number of layers**: 6
+- **Number of attention heads**: 4
+- **Vocabulary size**: 50304
+- **Max position embeddings**: 2048
 
 ## Parameters
 
-- **Total parameters**: ~195,072
-- **Trainable parameters**: ~195,072
+- **Total parameters**: ~14,067,712
+- **Trainable parameters**: ~14,067,712
 
 ## Usage
 

config.json

@@ -9,15 +9,14 @@
   "eos_token_id": 0,
   "hidden_act": "gelu",
   "hidden_dropout": 0.0,
-  "hidden_size": 64,
+  "hidden_size": 128,
   "initializer_range": 0.02,
-  "intermediate_size": 128,
+  "intermediate_size": 512,
   "layer_norm_eps": 1e-05,
-  "max_position_embeddings": 512,
+  "max_position_embeddings": 2048,
   "model_type": "gpt_neox",
-  "num_attention_heads": 2,
-  "num_hidden_layers": 2,
-  "num_key_value_heads": 2,
+  "num_attention_heads": 4,
+  "num_hidden_layers": 6,
   "partial_rotary_factor": 0.25,
   "rope_scaling": null,
   "rope_theta": 10000,
@@ -28,5 +27,5 @@
   "transformers_version": "4.54.0",
   "use_cache": true,
   "use_parallel_residual": true,
-  "vocab_size": 1000
+  "vocab_size": 50304
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:58e8409da8ee6ad38c51e0c68718a4dd271ebaf25472b9ddc014acc952361db5
-size 783264
+oid sha256:d2fc323f05723792edad751ae5ed494357fd1fcd77dad5f8736b3007076df64b
+size 56279344