Upload tiny random Llama-3.3 model (seed 42)

README.md

@@ -16,24 +16,24 @@ This is a tiny random version of the meta-llama/Llama-3.3-70B-Instruct model, cr
 
 - **Base model**: meta-llama/Llama-3.3-70B-Instruct
 - **Seed**: 42
-- **Hidden size**: 128
-- **Number of layers**: 4
-- **Number of attention heads**: 4
+- **Hidden size**: 768
+- **Number of layers**: 12
+- **Number of attention heads**: 12
 - **Vocabulary size**: 128256
 - **Max position embeddings**: 2048
 
 ## Parameters
 
-- **Total parameters**: ~62,031,104
-- **Trainable parameters**: ~62,031,104
+- **Total parameters**: ~272,517,888
+- **Trainable parameters**: ~272,517,888
 
 ## Usage
 
 ```python
-from transformers import AutoModelForSequenceClassification, AutoTokenizer
+from transformers import AutoModelForCausalLM, AutoTokenizer
 
 # Load model and tokenizer
-model = AutoModelForSequenceClassification.from_pretrained("AlignmentResearch/Llama-3.3-Tiny-Instruct")
+model = AutoModelForCausalLM.from_pretrained("AlignmentResearch/Llama-3.3-Tiny-Instruct")
 tokenizer = AutoTokenizer.from_pretrained("AlignmentResearch/Llama-3.3-Tiny-Instruct")
 
 # Generate text (note: this model has random weights!)

config.json

@@ -1,6 +1,6 @@
 {
   "architectures": [
-    "LlamaForSequenceClassification"
+    "LlamaForCausalLM"
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
@@ -10,23 +10,17 @@
     128008,
     128009
   ],
-  "head_dim": 128,
+  "head_dim": 64,
   "hidden_act": "silu",
-  "hidden_size": 128,
-  "id2label": {
-    "0": "LABEL_0"
-  },
+  "hidden_size": 768,
   "initializer_range": 0.02,
-  "intermediate_size": 28672,
-  "label2id": {
-    "LABEL_0": 0
-  },
+  "intermediate_size": 2048,
   "max_position_embeddings": 2048,
   "mlp_bias": false,
   "model_type": "llama",
-  "num_attention_heads": 4,
-  "num_hidden_layers": 4,
-  "num_key_value_heads": 8,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "num_key_value_heads": 4,
   "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
   "rope_scaling": {
@@ -38,7 +32,7 @@
   },
   "rope_theta": 500000.0,
   "tie_word_embeddings": false,
-  "torch_dtype": "bfloat16",
+  "torch_dtype": "float32",
   "transformers_version": "4.54.1",
   "use_cache": true,
   "vocab_size": 128256

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:01469a10977f03f5da4ae178136135866166b2b291237bddb6e7d8a569d823bc
-size 124066768
+oid sha256:51111d7052986501f4f8976ed86e6c838a273ddea1d9d8ce12d92d05c4d75c80
+size 1090083944