End of training

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6396043974964415e-06,
-    "train_runtime": 242.7438,
+    "train_loss": 9.608212667444605e-06,
+    "train_runtime": 236.596,
     "train_samples": 160,
-    "train_samples_per_second": 1.318,
-    "train_steps_per_second": 0.082
+    "train_samples_per_second": 1.353,
+    "train_steps_per_second": 0.085
 }

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:cf61bee1b4e1e36a71393f25392a102a4a6e17fb88aa02162c42187bee5de2b0
+oid sha256:98aaa78cbd212a7b2c406a9c5a8ed0d4e2b7d064024e071ba0ca4cba70dd72e9
 size 1976163472

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 1.6396043974964415e-06,
-    "train_runtime": 242.7438,
+    "train_loss": 9.608212667444605e-06,
+    "train_runtime": 236.596,
     "train_samples": 160,
-    "train_samples_per_second": 1.318,
-    "train_steps_per_second": 0.082
+    "train_samples_per_second": 1.353,
+    "train_steps_per_second": 0.085
 }

trainer_state.json

@@ -10,203 +10,203 @@
   "is_world_process_zero": true,
   "log_history": [
     {
-      "completion_length": 475.9375,
+      "completion_length": 464.59375,
       "epoch": 0.1,
-      "grad_norm": 13.196046829223633,
+      "grad_norm": 3.771817684173584,
       "kl": 0.0,
       "learning_rate": 4.965903258506806e-07,
-      "loss": 0.0,
-      "reward": 0.7955442824168131,
-      "reward_std": 0.9189272435032763,
-      "rewards/concensus_correctness_reward_func": 0.015375000424683094,
+      "loss": -0.0,
+      "reward": 1.3312862285529263,
+      "reward_std": 1.4614436527481303,
+      "rewards/concensus_correctness_reward_func": 0.6396249998360872,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.43826302388333715,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.32287997257662937,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.09190624766051769,
+      "rewards/xmlcount_reward_func": -0.00621874094940722,
       "step": 2
     },
     {
-      "completion_length": 502.71875,
+      "completion_length": 433.0,
       "epoch": 0.2,
-      "grad_norm": 3.139007806777954,
-      "kl": 0.0008146132586261956,
+      "grad_norm": 8.307954788208008,
+      "kl": 0.0008431348796875682,
       "learning_rate": 4.698684378016222e-07,
       "loss": 0.0,
-      "reward": 1.7969453744590282,
-      "reward_std": 1.9025329377036542,
-      "rewards/concensus_correctness_reward_func": 0.7559999991208315,
+      "reward": 0.8422680057119578,
+      "reward_std": 0.8241602857597172,
+      "rewards/concensus_correctness_reward_func": 0.06806250149384141,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.44357036444125697,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.5060180088039488,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.3473750064149499,
+      "rewards/xmlcount_reward_func": 0.0806875042617321,
       "step": 4
     },
     {
-      "completion_length": 505.0,
+      "completion_length": 439.46875,
       "epoch": 0.3,
-      "grad_norm": 4.405644416809082,
-      "kl": 0.0010104171597049572,
+      "grad_norm": 4.599847793579102,
+      "kl": 0.0011561252249521203,
       "learning_rate": 4.193203929064353e-07,
       "loss": 0.0,
-      "reward": 0.591551945079118,
-      "reward_std": 0.9647155045822728,
-      "rewards/concensus_correctness_reward_func": 0.0286250002682209,
+      "reward": 0.5898456154973246,
+      "reward_std": 0.7469334896450164,
+      "rewards/concensus_correctness_reward_func": 0.04025000147521496,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.34377070574555546,
+      "rewards/final_correctness_reward_func": 0.0625,
+      "rewards/question_recreation_reward_func": 0.3955018795095384,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.031656251987442374,
+      "rewards/xmlcount_reward_func": 0.029093743301928043,
       "step": 6
     },
     {
-      "completion_length": 440.9375,
+      "completion_length": 470.15625,
       "epoch": 0.4,
-      "grad_norm": 3.0010061264038086,
-      "kl": 0.0010926400391326752,
+      "grad_norm": 4.1406779289245605,
+      "kl": 0.0013139892726030666,
       "learning_rate": 3.5042385616324236e-07,
       "loss": 0.0,
-      "reward": 1.4674718617461622,
-      "reward_std": 1.8812191839097068,
-      "rewards/concensus_correctness_reward_func": 0.6514374995604157,
-      "rewards/consensus_reward_func": 0.125,
+      "reward": 2.326523531228304,
+      "reward_std": 2.568515565712005,
+      "rewards/concensus_correctness_reward_func": 1.249687500006985,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.125,
-      "rewards/question_recreation_reward_func": 0.5640343870036304,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.5765235247090459,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.0019999954383820295,
+      "rewards/xmlcount_reward_func": 0.18781250482425094,
       "step": 8
     },
     {
-      "completion_length": 496.25,
+      "completion_length": 501.90625,
       "epoch": 0.5,
-      "grad_norm": 4.053443431854248,
-      "kl": 0.0012771758265444078,
+      "grad_norm": 11.885337829589844,
+      "kl": 0.0014972705957916332,
       "learning_rate": 2.706448363680831e-07,
       "loss": 0.0,
-      "reward": 0.827742992900312,
-      "reward_std": 0.7718921254636371,
-      "rewards/concensus_correctness_reward_func": 0.01575000025331974,
+      "reward": 0.8391213522991166,
+      "reward_std": 0.6983874476718483,
+      "rewards/concensus_correctness_reward_func": 0.0078125,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.493430501781404,
-      "rewards/soft_format_reward_func": 0.015625,
+      "rewards/final_correctness_reward_func": 0.125,
+      "rewards/question_recreation_reward_func": 0.4944338685600087,
+      "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.17793749272823334,
+      "rewards/xmlcount_reward_func": 0.14937500841915607,
       "step": 10
     },
     {
-      "completion_length": 415.125,
+      "completion_length": 393.78125,
       "epoch": 0.6,
-      "grad_norm": 4.401289939880371,
-      "kl": 0.002079287798551377,
+      "grad_norm": 4.350698471069336,
+      "kl": 0.0017508711171103641,
       "learning_rate": 1.886286282148002e-07,
       "loss": 0.0,
-      "reward": 0.8090146852191538,
-      "reward_std": 0.7232096190564334,
-      "rewards/concensus_correctness_reward_func": 0.05937499925494194,
-      "rewards/consensus_reward_func": 0.0,
+      "reward": 1.0800049612298608,
+      "reward_std": 0.7751552746631205,
+      "rewards/concensus_correctness_reward_func": 0.08131249994039536,
+      "rewards/consensus_reward_func": 0.125,
       "rewards/cumulative_reward_2": 0.0,
       "rewards/final_correctness_reward_func": 0.0625,
-      "rewards/question_recreation_reward_func": 0.46720217959955335,
+      "rewards/question_recreation_reward_func": 0.5190674865152687,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.21993750659748912,
+      "rewards/xmlcount_reward_func": 0.29212500201538205,
       "step": 12
     },
     {
-      "completion_length": 460.84375,
+      "completion_length": 419.4375,
       "epoch": 0.7,
-      "grad_norm": 4.648487567901611,
-      "kl": 0.0015256664191838354,
+      "grad_norm": 4.202226161956787,
+      "kl": 0.00170601988065755,
       "learning_rate": 1.1326296046939333e-07,
       "loss": 0.0,
-      "reward": 1.8760306481271982,
-      "reward_std": 2.065487967338413,
-      "rewards/concensus_correctness_reward_func": 0.7446249965578318,
-      "rewards/consensus_reward_func": 0.1875,
+      "reward": 0.8355698073282838,
+      "reward_std": 1.1546667020302266,
+      "rewards/concensus_correctness_reward_func": 0.026750000193715096,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.3125,
-      "rewards/question_recreation_reward_func": 0.48543692484963685,
+      "rewards/final_correctness_reward_func": 0.1875,
+      "rewards/question_recreation_reward_func": 0.3973510548239574,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.145968749653548,
+      "rewards/xmlcount_reward_func": 0.16146875568665564,
       "step": 14
     },
     {
-      "completion_length": 425.03125,
+      "completion_length": 409.96875,
       "epoch": 0.8,
-      "grad_norm": 5.013484954833984,
-      "kl": 0.0017636593383940635,
+      "grad_norm": 5.176243305206299,
+      "kl": 0.0018919681824627332,
       "learning_rate": 5.271487265090163e-08,
       "loss": 0.0,
-      "reward": 2.299894895637408,
-      "reward_std": 2.430723395809764,
-      "rewards/concensus_correctness_reward_func": 1.298812499269843,
-      "rewards/consensus_reward_func": 0.0625,
+      "reward": 3.2867514765821397,
+      "reward_std": 2.303560282511171,
+      "rewards/concensus_correctness_reward_func": 2.109374998137355,
+      "rewards/consensus_reward_func": 0.25,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.1875,
-      "rewards/question_recreation_reward_func": 0.5421449297573417,
+      "rewards/final_correctness_reward_func": 0.375,
+      "rewards/question_recreation_reward_func": 0.43631404580082744,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.20893750712275505,
+      "rewards/xmlcount_reward_func": 0.11606249446049333,
       "step": 16
     },
     {
-      "completion_length": 367.09375,
+      "completion_length": 441.5,
       "epoch": 0.9,
-      "grad_norm": 5.056985378265381,
-      "kl": 0.0018181976101914188,
+      "grad_norm": 3.5152201652526855,
+      "kl": 0.08204289342575066,
       "learning_rate": 1.3545689574841341e-08,
-      "loss": 0.0,
-      "reward": 1.2346743869129568,
-      "reward_std": 0.7061302822985454,
-      "rewards/concensus_correctness_reward_func": 0.0,
-      "rewards/consensus_reward_func": 0.0,
+      "loss": 0.0001,
+      "reward": 1.3989268662407994,
+      "reward_std": 1.7090664645656943,
+      "rewards/concensus_correctness_reward_func": 0.6792499981820583,
+      "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.4375,
-      "rewards/question_recreation_reward_func": 0.41276813857257366,
-      "rewards/soft_format_reward_func": 0.0,
+      "rewards/final_correctness_reward_func": 0.25,
+      "rewards/question_recreation_reward_func": 0.3446768613066524,
+      "rewards/soft_format_reward_func": 0.015625,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.38440624717622995,
+      "rewards/xmlcount_reward_func": 0.04687500174622983,
       "step": 18
     },
     {
-      "completion_length": 383.40625,
+      "completion_length": 351.71875,
       "epoch": 1.0,
-      "grad_norm": 15.612112998962402,
-      "kl": 0.0021761340685770847,
+      "grad_norm": 10.250398635864258,
+      "kl": 0.003975560422986746,
       "learning_rate": 0.0,
       "loss": 0.0,
-      "reward": 1.698025934631005,
-      "reward_std": 1.2603743245708756,
-      "rewards/concensus_correctness_reward_func": 0.625,
+      "reward": 0.8882176700863056,
+      "reward_std": 0.7567949839867651,
+      "rewards/concensus_correctness_reward_func": 0.1001875028014183,
       "rewards/consensus_reward_func": 0.0625,
       "rewards/cumulative_reward_2": 0.0,
-      "rewards/final_correctness_reward_func": 0.0,
-      "rewards/question_recreation_reward_func": 0.46149466722272336,
+      "rewards/final_correctness_reward_func": 0.3125,
+      "rewards/question_recreation_reward_func": 0.30937393981730565,
       "rewards/soft_format_reward_func": 0.0,
       "rewards/strict_format_reward_func": 0.0,
-      "rewards/xmlcount_reward_func": 0.5490312427282333,
+      "rewards/xmlcount_reward_func": 0.1036562561057508,
       "step": 20
     },
     {
       "epoch": 1.0,
       "step": 20,
       "total_flos": 0.0,
-      "train_loss": 1.6396043974964415e-06,
-      "train_runtime": 242.7438,
-      "train_samples_per_second": 1.318,
-      "train_steps_per_second": 0.082
+      "train_loss": 9.608212667444605e-06,
+      "train_runtime": 236.596,
+      "train_samples_per_second": 1.353,
+      "train_steps_per_second": 0.085
     }
   ],
   "logging_steps": 2,