Model save

README.md

@@ -1,32 +1,29 @@
 ---
 license: apache-2.0
-base_model: alignment-handbook/zephyr-7b-sft-full
+base_model: plaguss/zephyr-7b-spin-iter0-v0
 tags:
-- alignment-handbook
 - generated_from_trainer
-datasets:
-- argilla/10k_prompts_SPIN_iter0_zephyr_top
 model-index:
-- name: outputs
+- name: zephyr-7b-spin-iter1-v0
   results: []
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
 should probably proofread and complete it, then remove this comment. -->
 
-# outputs
+# zephyr-7b-spin-iter1-v0
 
-This model is a fine-tuned version of [alignment-handbook/zephyr-7b-sft-full](https://huggingface.co/alignment-handbook/zephyr-7b-sft-full) on the argilla/10k_prompts_SPIN_iter0_zephyr_top dataset.
+This model is a fine-tuned version of [plaguss/zephyr-7b-spin-iter0-v0](https://huggingface.co/plaguss/zephyr-7b-spin-iter0-v0) on the None dataset.
 It achieves the following results on the evaluation set:
-- Loss: 0.2359
-- Rewards/real: 1.3255
-- Rewards/generated: -0.8966
+- Loss: 0.0831
+- Rewards/real: 1.3037
+- Rewards/generated: -5.4434
 - Rewards/accuracies: 0.9792
-- Rewards/margins: 2.2221
-- Logps/generated: -309.8145
-- Logps/real: -304.9670
-- Logits/generated: -2.7558
-- Logits/real: -2.7547
+- Rewards/margins: 6.7471
+- Logps/generated: -545.0309
+- Logps/real: -272.3726
+- Logits/generated: -2.6844
+- Logits/real: -2.7197
 
 ## Model description
 
@@ -63,8 +60,10 @@ The following hyperparameters were used during training:
 
 | Training Loss | Epoch | Step | Validation Loss | Rewards/real | Rewards/generated | Rewards/accuracies | Rewards/margins | Logps/generated | Logps/real | Logits/generated | Logits/real |
 |:-------------:|:-----:|:----:|:---------------:|:------------:|:-----------------:|:------------------:|:---------------:|:---------------:|:----------:|:----------------:|:-----------:|
-| 0.3011        | 0.96  | 25   | 0.2442          | 1.1606       | -0.9851           | 0.9792             | 2.1457          | -310.6989       | -306.6157  | -2.7644          | -2.7641     |
-| 0.0376        | 1.92  | 50   | 0.2359          | 1.3255       | -0.8966           | 0.9792             | 2.2221          | -309.8145       | -304.9670  | -2.7558          | -2.7547     |
+| 0.1827        | 0.49  | 25   | 0.1651          | 0.1714       | -3.3650           | 0.9688             | 3.5364          | -524.2469       | -283.6962  | -2.7482          | -2.7944     |
+| 0.0462        | 0.97  | 50   | 0.0835          | 1.4823       | -4.4998           | 1.0                | 5.9821          | -535.5947       | -270.5871  | -2.6963          | -2.7356     |
+| 0.0047        | 1.46  | 75   | 0.0837          | 1.3725       | -5.2500           | 0.9896             | 6.6225          | -543.0965       | -271.6846  | -2.6847          | -2.7211     |
+| 0.0034        | 1.94  | 100  | 0.0831          | 1.3037       | -5.4434           | 0.9792             | 6.7471          | -545.0309       | -272.3726  | -2.6844          | -2.7197     |
 
 
 ### Framework versions

all_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "train_loss": 0.2113667087486157,
-    "train_runtime": 1162.1581,
-    "train_samples": 1648,
-    "train_samples_per_second": 2.836,
+    "epoch": 1.98,
+    "train_loss": 0.09802283835140806,
+    "train_runtime": 2267.6641,
+    "train_samples": 3296,
+    "train_samples_per_second": 2.907,
     "train_steps_per_second": 0.045
 }

model-00001-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b57c4c82684914d3b1f9f2f7631034d52a02ac14d7bfbe136280e383f32cc294
+oid sha256:692333829b71c45499efd2758619021ab865b5466e18c39d54b0513f17b21c96
 size 4943162336

model-00002-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:456848aaa337d93707cfa3b47a7be5953b20a81cbf3a351293582933f1398781
+oid sha256:d2776af7f7fd4970ee2109cd4a99f755cc91babdbb7b300d9c7f6aff27b47589
 size 4999819336

model-00003-of-00003.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:da6ba25be6087ce9a2fcadf2792cab1d36b2e3b4d2ebe988876d2b0f3bb733ac
+oid sha256:1c9d190f377227e3e89b7c96490ab1eac0822345aaf018caa80cb93af167bbc9
 size 4540516344

train_results.json

@@ -1,8 +1,8 @@
 {
-    "epoch": 2.0,
-    "train_loss": 0.2113667087486157,
-    "train_runtime": 1162.1581,
-    "train_samples": 1648,
-    "train_samples_per_second": 2.836,
+    "epoch": 1.98,
+    "train_loss": 0.09802283835140806,
+    "train_runtime": 2267.6641,
+    "train_samples": 3296,
+    "train_samples_per_second": 2.907,
     "train_steps_per_second": 0.045
 }

trainer_state.json

@@ -1,20 +1,20 @@
 {
   "best_metric": null,
   "best_model_checkpoint": null,
-  "epoch": 2.0,
+  "epoch": 1.9805825242718447,
   "eval_steps": 25,
-  "global_step": 52,
+  "global_step": 102,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
     {
-      "epoch": 0.04,
-      "learning_rate": 8.333333333333333e-08,
-      "logits/generated": -2.788468599319458,
-      "logits/real": -2.8911099433898926,
-      "logps/generated": -226.66921997070312,
-      "logps/real": -283.6243896484375,
+      "epoch": 0.02,
+      "learning_rate": 4.545454545454545e-08,
+      "logits/generated": -2.706744432449341,
+      "logits/real": -2.8291945457458496,
+      "logps/generated": -517.0836791992188,
+      "logps/real": -317.913818359375,
       "loss": 0.6931,
       "rewards/accuracies": 0.0,
       "rewards/generated": 0.0,
@@ -23,119 +23,221 @@
       "step": 1
     },
     {
-      "epoch": 0.38,
-      "learning_rate": 4.5652173913043473e-07,
-      "logits/generated": -2.8515138626098633,
-      "logits/real": -2.8768396377563477,
-      "logps/generated": -354.09619140625,
-      "logps/real": -350.52911376953125,
-      "loss": 0.522,
-      "rewards/accuracies": 0.7569444179534912,
-      "rewards/generated": -0.21337264776229858,
-      "rewards/margins": 0.5563015341758728,
-      "rewards/real": 0.3429288864135742,
+      "epoch": 0.19,
+      "learning_rate": 4.545454545454545e-07,
+      "logits/generated": -2.71917462348938,
+      "logits/real": -2.7550785541534424,
+      "logps/generated": -531.1511840820312,
+      "logps/real": -286.0602111816406,
+      "loss": 0.5456,
+      "rewards/accuracies": 0.7291666865348816,
+      "rewards/generated": -0.04339843988418579,
+      "rewards/margins": 0.3565465211868286,
+      "rewards/real": 0.3131480813026428,
       "step": 10
     },
     {
-      "epoch": 0.77,
-      "learning_rate": 3.478260869565217e-07,
-      "logits/generated": -2.792628049850464,
-      "logits/real": -2.7778868675231934,
-      "logps/generated": -351.04638671875,
-      "logps/real": -327.13482666015625,
-      "loss": 0.3011,
-      "rewards/accuracies": 0.90625,
-      "rewards/generated": -0.7889599800109863,
-      "rewards/margins": 1.546514868736267,
-      "rewards/real": 0.7575550675392151,
+      "epoch": 0.39,
+      "learning_rate": 4.5054945054945056e-07,
+      "logits/generated": -2.830758571624756,
+      "logits/real": -2.861502170562744,
+      "logps/generated": -586.2496337890625,
+      "logps/real": -281.56402587890625,
+      "loss": 0.1827,
+      "rewards/accuracies": 0.981249988079071,
+      "rewards/generated": -1.7483899593353271,
+      "rewards/margins": 2.579699993133545,
+      "rewards/real": 0.8313096761703491,
       "step": 20
     },
     {
-      "epoch": 0.96,
-      "eval_logits/generated": -2.764375686645508,
-      "eval_logits/real": -2.7640507221221924,
-      "eval_logps/generated": -310.69891357421875,
-      "eval_logps/real": -306.61572265625,
-      "eval_loss": 0.24416939914226532,
-      "eval_rewards/accuracies": 0.9791666865348816,
-      "eval_rewards/generated": -0.9850902557373047,
-      "eval_rewards/margins": 2.14570689201355,
-      "eval_rewards/real": 1.1606166362762451,
-      "eval_runtime": 27.6861,
-      "eval_samples_per_second": 6.646,
-      "eval_steps_per_second": 0.217,
+      "epoch": 0.49,
+      "eval_logits/generated": -2.7481918334960938,
+      "eval_logits/real": -2.7943999767303467,
+      "eval_logps/generated": -524.2468872070312,
+      "eval_logps/real": -283.6961669921875,
+      "eval_loss": 0.16513165831565857,
+      "eval_rewards/accuracies": 0.96875,
+      "eval_rewards/generated": -3.365018129348755,
+      "eval_rewards/margins": 3.536374092102051,
+      "eval_rewards/real": 0.17135602235794067,
+      "eval_runtime": 54.5373,
+      "eval_samples_per_second": 6.748,
+      "eval_steps_per_second": 0.22,
       "step": 25
     },
     {
-      "epoch": 1.15,
-      "learning_rate": 2.391304347826087e-07,
-      "logits/generated": -2.7530007362365723,
-      "logits/real": -2.734692096710205,
-      "logps/generated": -310.22607421875,
-      "logps/real": -306.02044677734375,
-      "loss": 0.1788,
-      "rewards/accuracies": 0.9312499761581421,
-      "rewards/generated": -1.791497826576233,
-      "rewards/margins": 3.7750840187072754,
-      "rewards/real": 1.9835857152938843,
+      "epoch": 0.58,
+      "learning_rate": 3.9560439560439557e-07,
+      "logits/generated": -2.7771615982055664,
+      "logits/real": -2.807366371154785,
+      "logps/generated": -499.19207763671875,
+      "logps/real": -301.99432373046875,
+      "loss": 0.1089,
+      "rewards/accuracies": 0.956250011920929,
+      "rewards/generated": -2.920013427734375,
+      "rewards/margins": 3.9550042152404785,
+      "rewards/real": 1.034990668296814,
       "step": 30
     },
     {
-      "epoch": 1.54,
-      "learning_rate": 1.3043478260869563e-07,
-      "logits/generated": -2.7655322551727295,
-      "logits/real": -2.776773691177368,
-      "logps/generated": -358.19403076171875,
-      "logps/real": -309.92767333984375,
-      "loss": 0.0384,
-      "rewards/accuracies": 0.9937499761581421,
-      "rewards/generated": -3.1963627338409424,
-      "rewards/margins": 6.872523307800293,
-      "rewards/real": 3.6761608123779297,
+      "epoch": 0.78,
+      "learning_rate": 3.4065934065934063e-07,
+      "logits/generated": -2.665315628051758,
+      "logits/real": -2.7098612785339355,
+      "logps/generated": -539.9364013671875,
+      "logps/real": -248.0373077392578,
+      "loss": 0.0698,
+      "rewards/accuracies": 0.96875,
+      "rewards/generated": -3.776944398880005,
+      "rewards/margins": 5.202493667602539,
+      "rewards/real": 1.4255495071411133,
       "step": 40
     },
     {
-      "epoch": 1.92,
-      "learning_rate": 2.1739130434782606e-08,
-      "logits/generated": -2.7564454078674316,
-      "logits/real": -2.7757363319396973,
-      "logps/generated": -357.3354797363281,
-      "logps/real": -296.8515930175781,
-      "loss": 0.0376,
-      "rewards/accuracies": 0.987500011920929,
-      "rewards/generated": -2.9351892471313477,
-      "rewards/margins": 6.2575507164001465,
-      "rewards/real": 3.322361707687378,
+      "epoch": 0.97,
+      "learning_rate": 2.857142857142857e-07,
+      "logits/generated": -2.6974306106567383,
+      "logits/real": -2.761044979095459,
+      "logps/generated": -644.1237182617188,
+      "logps/real": -292.7959899902344,
+      "loss": 0.0462,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -4.7168684005737305,
+      "rewards/margins": 6.229194164276123,
+      "rewards/real": 1.5123255252838135,
       "step": 50
     },
     {
-      "epoch": 1.92,
-      "eval_logits/generated": -2.7557647228240967,
-      "eval_logits/real": -2.7546520233154297,
-      "eval_logps/generated": -309.8145446777344,
-      "eval_logps/real": -304.967041015625,
-      "eval_loss": 0.23592980206012726,
-      "eval_rewards/accuracies": 0.9791666865348816,
-      "eval_rewards/generated": -0.8966498374938965,
-      "eval_rewards/margins": 2.2221336364746094,
-      "eval_rewards/real": 1.3254839181900024,
-      "eval_runtime": 27.8272,
-      "eval_samples_per_second": 6.612,
-      "eval_steps_per_second": 0.216,
+      "epoch": 0.97,
+      "eval_logits/generated": -2.6962978839874268,
+      "eval_logits/real": -2.7355997562408447,
+      "eval_logps/generated": -535.5946655273438,
+      "eval_logps/real": -270.58709716796875,
+      "eval_loss": 0.08352091163396835,
+      "eval_rewards/accuracies": 1.0,
+      "eval_rewards/generated": -4.499805450439453,
+      "eval_rewards/margins": 5.9820709228515625,
+      "eval_rewards/real": 1.4822653532028198,
+      "eval_runtime": 54.7978,
+      "eval_samples_per_second": 6.716,
+      "eval_steps_per_second": 0.219,
       "step": 50
     },
     {
-      "epoch": 2.0,
-      "step": 52,
+      "epoch": 1.17,
+      "learning_rate": 2.3076923076923078e-07,
+      "logits/generated": -2.7394375801086426,
+      "logits/real": -2.751296043395996,
+      "logps/generated": -666.791748046875,
+      "logps/real": -282.9411926269531,
+      "loss": 0.0094,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -6.813584804534912,
+      "rewards/margins": 9.226309776306152,
+      "rewards/real": 2.4127261638641357,
+      "step": 60
+    },
+    {
+      "epoch": 1.36,
+      "learning_rate": 1.7582417582417584e-07,
+      "logits/generated": -2.7095775604248047,
+      "logits/real": -2.76279616355896,
+      "logps/generated": -612.8692626953125,
+      "logps/real": -276.9359130859375,
+      "loss": 0.0047,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -7.154807090759277,
+      "rewards/margins": 9.882351875305176,
+      "rewards/real": 2.727543592453003,
+      "step": 70
+    },
+    {
+      "epoch": 1.46,
+      "eval_logits/generated": -2.6846535205841064,
+      "eval_logits/real": -2.7210752964019775,
+      "eval_logps/generated": -543.0964965820312,
+      "eval_logps/real": -271.6846008300781,
+      "eval_loss": 0.08372422307729721,
+      "eval_rewards/accuracies": 0.9895833134651184,
+      "eval_rewards/generated": -5.249976634979248,
+      "eval_rewards/margins": 6.622490406036377,
+      "eval_rewards/real": 1.3725138902664185,
+      "eval_runtime": 56.1231,
+      "eval_samples_per_second": 6.557,
+      "eval_steps_per_second": 0.214,
+      "step": 75
+    },
+    {
+      "epoch": 1.55,
+      "learning_rate": 1.2087912087912088e-07,
+      "logits/generated": -2.6911048889160156,
+      "logits/real": -2.730372667312622,
+      "logps/generated": -622.9490966796875,
+      "logps/real": -264.1090087890625,
+      "loss": 0.0063,
+      "rewards/accuracies": 0.9937499761581421,
+      "rewards/generated": -7.1237335205078125,
+      "rewards/margins": 9.698349952697754,
+      "rewards/real": 2.5746169090270996,
+      "step": 80
+    },
+    {
+      "epoch": 1.75,
+      "learning_rate": 6.593406593406594e-08,
+      "logits/generated": -2.6824026107788086,
+      "logits/real": -2.712606906890869,
+      "logps/generated": -557.6444702148438,
+      "logps/real": -256.166259765625,
+      "loss": 0.0079,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -7.720358371734619,
+      "rewards/margins": 10.142163276672363,
+      "rewards/real": 2.4218039512634277,
+      "step": 90
+    },
+    {
+      "epoch": 1.94,
+      "learning_rate": 1.098901098901099e-08,
+      "logits/generated": -2.727700710296631,
+      "logits/real": -2.769242525100708,
+      "logps/generated": -596.8726806640625,
+      "logps/real": -262.6658020019531,
+      "loss": 0.0034,
+      "rewards/accuracies": 1.0,
+      "rewards/generated": -7.486897945404053,
+      "rewards/margins": 9.872056007385254,
+      "rewards/real": 2.385158061981201,
+      "step": 100
+    },
+    {
+      "epoch": 1.94,
+      "eval_logits/generated": -2.6844117641448975,
+      "eval_logits/real": -2.7196710109710693,
+      "eval_logps/generated": -545.0308837890625,
+      "eval_logps/real": -272.3725891113281,
+      "eval_loss": 0.08312664180994034,
+      "eval_rewards/accuracies": 0.9791666865348816,
+      "eval_rewards/generated": -5.443421840667725,
+      "eval_rewards/margins": 6.747138500213623,
+      "eval_rewards/real": 1.3037166595458984,
+      "eval_runtime": 56.0968,
+      "eval_samples_per_second": 6.56,
+      "eval_steps_per_second": 0.214,
+      "step": 100
+    },
+    {
+      "epoch": 1.98,
+      "step": 102,
       "total_flos": 0.0,
-      "train_loss": 0.2113667087486157,
-      "train_runtime": 1162.1581,
-      "train_samples_per_second": 2.836,
+      "train_loss": 0.09802283835140806,
+      "train_runtime": 2267.6641,
+      "train_samples_per_second": 2.907,
       "train_steps_per_second": 0.045
     }
   ],
   "logging_steps": 10,
-  "max_steps": 52,
+  "max_steps": 102,
   "num_input_tokens_seen": 0,
   "num_train_epochs": 2,
   "save_steps": 500,