Model save

README.md

@@ -26,7 +26,7 @@ print(output["generated_text"])
 
 ## Training procedure
 
-[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/luojing020713-siat/huggingface/runs/h0e5120i) 
+[<img src="https://raw.githubusercontent.com/wandb/assets/main/wandb-github-badge-28.svg" alt="Visualize in Weights & Biases" width="150" height="24"/>](https://wandb.ai/luojing020713-siat/huggingface/runs/i24cg4sm) 
 
 
 This model was trained with GRPO, a method introduced in [DeepSeekMath: Pushing the Limits of Mathematical Reasoning in Open Language Models](https://huggingface.co/papers/2402.03300).

all_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.019510615981001962,
-    "train_runtime": 40590.3738,
+    "train_loss": 0.005782647762212089,
+    "train_runtime": 41644.7787,
     "train_samples": 7500,
-    "train_samples_per_second": 0.554,
+    "train_samples_per_second": 0.54,
     "train_steps_per_second": 0.004
 }

model-00001-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:1eea86c220d76b32b2b9cc665dd8fec3d8a18681bdb03f0c0e420a3b5c108fbc
+oid sha256:3a295261b7109e67173c7e7151d15f10df25b1a1a0842779d40df942c3caccdf
 size 4957560304

model-00002-of-00002.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e40b6202a36e1c10cff24667c72d0716696a37d80ba3fd243368c348d390b70b
+oid sha256:7065c71486bc879a9100559062b65218d55d9b73b619409d90ec664d49b7ce79
 size 1836696752

train_results.json

@@ -1,8 +1,8 @@
 {
     "total_flos": 0.0,
-    "train_loss": 0.019510615981001962,
-    "train_runtime": 40590.3738,
+    "train_loss": 0.005782647762212089,
+    "train_runtime": 41644.7787,
     "train_samples": 7500,
-    "train_samples_per_second": 0.554,
+    "train_samples_per_second": 0.54,
     "train_steps_per_second": 0.004
 }

trainer_state.json

@@ -10,552 +10,516 @@
   "log_history": [
     {
       "clip_ratio": 0.0,
-      "completion_length": 598.53515625,
+      "completion_length": 648.7421875,
       "epoch": 0.017057569296375266,
-      "grad_norm": 0.21371020376682281,
+      "grad_norm": 0.06589560955762863,
       "kl": 0.0,
       "learning_rate": 1.6666666666666665e-07,
-      "loss": 0.0226,
-      "reward": 0.6796875,
-      "reward_std": 0.25411649234592915,
-      "rewards/accuracy_reward": 0.53125,
-      "rewards/format_reward": 0.1484375,
-      "rewards/reflection_reward_pos": 0.0,
+      "loss": 0.0062,
+      "reward": 0.703125,
+      "reward_std": 0.0883883461356163,
+      "rewards/accuracy_reward": 0.69921875,
+      "rewards/reflection_reward_pos": 0.00390625,
       "step": 1
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 573.52734375,
+      "completion_length": 648.5087890625,
       "epoch": 0.08528784648187633,
-      "grad_norm": 0.22427839040756226,
-      "kl": 0.00026004016399383545,
+      "grad_norm": 0.06861326843500137,
+      "kl": 0.00011852383613586426,
       "learning_rate": 8.333333333333334e-07,
-      "loss": 0.0238,
-      "reward": 0.6337890625,
-      "reward_std": 0.24444902036339045,
-      "rewards/accuracy_reward": 0.4833984375,
-      "rewards/format_reward": 0.1494140625,
-      "rewards/reflection_reward_pos": 0.0009765625,
+      "loss": -0.0011,
+      "reward": 0.6640625,
+      "reward_std": 0.09115048055537045,
+      "rewards/accuracy_reward": 0.6640625,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 5
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 530.23671875,
+      "completion_length": 638.33359375,
       "epoch": 0.17057569296375266,
-      "grad_norm": 0.24708664417266846,
-      "kl": 0.0064833641052246095,
+      "grad_norm": 0.06051831692457199,
+      "kl": 0.00015828609466552734,
       "learning_rate": 1.6666666666666669e-06,
-      "loss": 0.0162,
-      "reward": 0.6828125,
-      "reward_std": 0.22759998859837652,
-      "rewards/accuracy_reward": 0.51171875,
-      "rewards/format_reward": 0.1703125,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.01,
+      "reward": 0.67734375,
+      "reward_std": 0.10275145107880235,
+      "rewards/accuracy_reward": 0.67734375,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 10
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 543.28359375,
+      "completion_length": 622.8640625,
       "epoch": 0.255863539445629,
-      "grad_norm": 0.20078016817569733,
-      "kl": 0.001663541793823242,
+      "grad_norm": 0.09521586447954178,
+      "kl": 0.0001518726348876953,
       "learning_rate": 2.5e-06,
-      "loss": 0.0205,
-      "reward": 0.68671875,
-      "reward_std": 0.22207571836188436,
-      "rewards/accuracy_reward": 0.51640625,
-      "rewards/format_reward": 0.16796875,
-      "rewards/reflection_reward_pos": 0.00234375,
+      "loss": 0.0024,
+      "reward": 0.71015625,
+      "reward_std": 0.12042912095785141,
+      "rewards/accuracy_reward": 0.709375,
+      "rewards/reflection_reward_pos": 0.00078125,
       "step": 15
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 515.36875,
+      "completion_length": 607.86640625,
       "epoch": 0.3411513859275053,
-      "grad_norm": 0.48252072930336,
-      "kl": 0.008718490600585938,
+      "grad_norm": 0.08084629476070404,
+      "kl": 0.00022783279418945313,
       "learning_rate": 2.9987834972573546e-06,
-      "loss": 0.0157,
-      "reward": 0.70234375,
-      "reward_std": 0.23533396841958165,
-      "rewards/accuracy_reward": 0.53046875,
-      "rewards/format_reward": 0.16875,
-      "rewards/reflection_reward_pos": 0.003125,
+      "loss": 0.0036,
+      "reward": 0.70546875,
+      "reward_std": 0.12042912160977721,
+      "rewards/accuracy_reward": 0.70390625,
+      "rewards/reflection_reward_pos": 0.0015625,
       "step": 20
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 563.7859375,
+      "completion_length": 645.02421875,
       "epoch": 0.42643923240938164,
-      "grad_norm": 0.25852227210998535,
-      "kl": 0.015869140625,
+      "grad_norm": 0.08875104784965515,
+      "kl": 0.00043997764587402346,
       "learning_rate": 2.9851204919417464e-06,
-      "loss": 0.0099,
-      "reward": 0.63984375,
-      "reward_std": 0.22870484348386527,
-      "rewards/accuracy_reward": 0.49140625,
-      "rewards/format_reward": 0.1453125,
-      "rewards/reflection_reward_pos": 0.003125,
+      "loss": 0.0049,
+      "reward": 0.65859375,
+      "reward_std": 0.11379999481141567,
+      "rewards/accuracy_reward": 0.65859375,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 25
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 568.0296875,
+      "completion_length": 670.86953125,
       "epoch": 0.511727078891258,
-      "grad_norm": 0.17840448021888733,
-      "kl": 0.017840576171875,
+      "grad_norm": 0.07565722614526749,
+      "kl": 0.0007645606994628907,
       "learning_rate": 2.956412726139078e-06,
-      "loss": 0.0123,
-      "reward": 0.6546875,
-      "reward_std": 0.23422911390662193,
-      "rewards/accuracy_reward": 0.51875,
-      "rewards/format_reward": 0.13515625,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.0067,
+      "reward": 0.67421875,
+      "reward_std": 0.10054174307733774,
+      "rewards/accuracy_reward": 0.67265625,
+      "rewards/reflection_reward_pos": 0.0015625,
       "step": 30
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 571.41328125,
+      "completion_length": 659.73203125,
       "epoch": 0.5970149253731343,
-      "grad_norm": 0.20260392129421234,
-      "kl": 0.022509765625,
+      "grad_norm": 0.07428640872240067,
+      "kl": 0.0010577201843261718,
       "learning_rate": 2.9129510189868974e-06,
-      "loss": 0.0141,
-      "reward": 0.64609375,
-      "reward_std": 0.19997863341122865,
-      "rewards/accuracy_reward": 0.48125,
-      "rewards/format_reward": 0.16328125,
-      "rewards/reflection_reward_pos": 0.0015625,
+      "loss": 0.0062,
+      "reward": 0.65,
+      "reward_std": 0.08838834529742598,
+      "rewards/accuracy_reward": 0.65,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 35
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 544.46328125,
+      "completion_length": 645.309375,
       "epoch": 0.6823027718550106,
-      "grad_norm": 0.3073066174983978,
-      "kl": 0.0449493408203125,
+      "grad_norm": 0.08162426203489304,
+      "kl": 0.0015665054321289062,
       "learning_rate": 2.8551756519155732e-06,
-      "loss": 0.0171,
-      "reward": 0.6875,
-      "reward_std": 0.21876115268096327,
-      "rewards/accuracy_reward": 0.50078125,
-      "rewards/format_reward": 0.1859375,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.0098,
+      "reward": 0.6921875,
+      "reward_std": 0.10385630577802658,
+      "rewards/accuracy_reward": 0.6921875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 40
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 570.12890625,
+      "completion_length": 655.0703125,
       "epoch": 0.767590618336887,
-      "grad_norm": 0.26227954030036926,
-      "kl": 0.091302490234375,
+      "grad_norm": 0.08563440293073654,
+      "kl": 0.0018520355224609375,
       "learning_rate": 2.7836719084521715e-06,
-      "loss": 0.0133,
-      "reward": 0.65390625,
-      "reward_std": 0.22428542636334897,
-      "rewards/accuracy_reward": 0.49140625,
-      "rewards/format_reward": 0.159375,
-      "rewards/reflection_reward_pos": 0.003125,
+      "loss": 0.0064,
+      "reward": 0.65859375,
+      "reward_std": 0.12484853798523546,
+      "rewards/accuracy_reward": 0.65859375,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 45
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 548.32421875,
+      "completion_length": 634.13515625,
       "epoch": 0.8528784648187633,
-      "grad_norm": 0.15913987159729004,
-      "kl": 0.046868896484375,
+      "grad_norm": 0.08154138922691345,
+      "kl": 0.0025421142578125,
       "learning_rate": 2.699164145105252e-06,
-      "loss": 0.0176,
-      "reward": 0.646875,
-      "reward_std": 0.24085824135690928,
-      "rewards/accuracy_reward": 0.46953125,
-      "rewards/format_reward": 0.17578125,
-      "rewards/reflection_reward_pos": 0.0015625,
+      "loss": 0.0071,
+      "reward": 0.6671875,
+      "reward_std": 0.11490484932437539,
+      "rewards/accuracy_reward": 0.66484375,
+      "rewards/reflection_reward_pos": 0.00234375,
       "step": 50
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 539.67109375,
+      "completion_length": 648.434375,
       "epoch": 0.9381663113006397,
-      "grad_norm": 0.34673067927360535,
-      "kl": 0.0633026123046875,
+      "grad_norm": 0.09137308597564697,
+      "kl": 0.0026458740234375,
       "learning_rate": 2.602508453394493e-06,
-      "loss": 0.0218,
-      "reward": 0.68203125,
-      "reward_std": 0.23975338581949474,
-      "rewards/accuracy_reward": 0.50390625,
-      "rewards/format_reward": 0.17578125,
-      "rewards/reflection_reward_pos": 0.00234375,
+      "loss": 0.0071,
+      "reward": 0.6796875,
+      "reward_std": 0.1303728088736534,
+      "rewards/accuracy_reward": 0.6796875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 55
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 533.0619819641113,
+      "completion_length": 622.5903686523437,
       "epoch": 1.0341151385927505,
-      "grad_norm": 0.17685599625110626,
-      "kl": 0.079449462890625,
+      "grad_norm": 0.10423822700977325,
+      "kl": 0.0027740478515625,
       "learning_rate": 2.4946839873611927e-06,
-      "loss": 0.0214,
-      "reward": 0.66875,
-      "reward_std": 0.223180572129786,
-      "rewards/accuracy_reward": 0.51640625,
-      "rewards/format_reward": 0.15078125,
-      "rewards/reflection_reward_pos": 0.0015625,
+      "loss": 0.0048,
+      "reward": 0.6671875,
+      "reward_std": 0.11932426644489169,
+      "rewards/accuracy_reward": 0.6671875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 60
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 531.99140625,
+      "completion_length": 608.39921875,
       "epoch": 1.1194029850746268,
-      "grad_norm": 0.2092791497707367,
-      "kl": 0.0752899169921875,
+      "grad_norm": 0.10257695615291595,
+      "kl": 0.0032474517822265623,
       "learning_rate": 2.3767830444148337e-06,
-      "loss": 0.0198,
-      "reward": 0.6484375,
-      "reward_std": 0.24306795094162226,
-      "rewards/accuracy_reward": 0.48828125,
-      "rewards/format_reward": 0.1546875,
-      "rewards/reflection_reward_pos": 0.00546875,
+      "loss": 0.0086,
+      "reward": 0.68515625,
+      "reward_std": 0.10496116010472178,
+      "rewards/accuracy_reward": 0.68515625,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 65
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 534.46328125,
+      "completion_length": 617.39140625,
       "epoch": 1.2046908315565032,
-      "grad_norm": 0.16183458268642426,
-      "kl": 0.06942138671875,
+      "grad_norm": 0.0765712708234787,
+      "kl": 0.0033966064453125,
       "learning_rate": 2.25e-06,
-      "loss": 0.0175,
-      "reward": 0.69609375,
-      "reward_std": 0.21765630040317774,
-      "rewards/accuracy_reward": 0.5046875,
-      "rewards/format_reward": 0.18984375,
-      "rewards/reflection_reward_pos": 0.0015625,
+      "loss": 0.005,
+      "reward": 0.68046875,
+      "reward_std": 0.10275145145133138,
+      "rewards/accuracy_reward": 0.68046875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 70
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 550.5828125,
+      "completion_length": 620.7875,
       "epoch": 1.2899786780383795,
-      "grad_norm": 0.31448841094970703,
-      "kl": 0.170635986328125,
+      "grad_norm": 0.1029704362154007,
+      "kl": 0.0029888153076171875,
       "learning_rate": 2.1156192081791355e-06,
-      "loss": 0.0202,
-      "reward": 0.6625,
-      "reward_std": 0.22980969864875078,
-      "rewards/accuracy_reward": 0.49765625,
-      "rewards/format_reward": 0.1609375,
-      "rewards/reflection_reward_pos": 0.00390625,
+      "loss": 0.0023,
+      "reward": 0.6828125,
+      "reward_std": 0.12595339212566614,
+      "rewards/accuracy_reward": 0.68203125,
+      "rewards/reflection_reward_pos": 0.00078125,
       "step": 75
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 531.7296875,
+      "completion_length": 615.33828125,
       "epoch": 1.375266524520256,
-      "grad_norm": 2.1899828910827637,
-      "kl": 0.155224609375,
+      "grad_norm": 0.08894416689872742,
+      "kl": 0.0030879974365234375,
       "learning_rate": 1.975001990702209e-06,
-      "loss": 0.0176,
-      "reward": 0.6984375,
-      "reward_std": 0.23422911493107676,
-      "rewards/accuracy_reward": 0.52890625,
-      "rewards/format_reward": 0.16875,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.0046,
+      "reward": 0.71484375,
+      "reward_std": 0.12042912067845464,
+      "rewards/accuracy_reward": 0.71484375,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 80
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 544.83046875,
+      "completion_length": 629.1375,
       "epoch": 1.4605543710021323,
-      "grad_norm": 0.4367416501045227,
-      "kl": 0.205291748046875,
+      "grad_norm": 0.08637527376413345,
+      "kl": 0.0033039093017578126,
       "learning_rate": 1.829572846368326e-06,
-      "loss": 0.0195,
-      "reward": 0.675,
-      "reward_std": 0.26074561905115845,
-      "rewards/accuracy_reward": 0.509375,
-      "rewards/format_reward": 0.16484375,
+      "loss": 0.0055,
+      "reward": 0.690625,
+      "reward_std": 0.11490484941750764,
+      "rewards/accuracy_reward": 0.68984375,
       "rewards/reflection_reward_pos": 0.00078125,
       "step": 85
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 552.22890625,
+      "completion_length": 652.8375,
       "epoch": 1.5458422174840085,
-      "grad_norm": 0.24191254377365112,
-      "kl": 0.123126220703125,
+      "grad_norm": 0.0866626650094986,
+      "kl": 0.0031810760498046874,
       "learning_rate": 1.6808050203829845e-06,
-      "loss": 0.0164,
-      "reward": 0.66015625,
-      "reward_std": 0.21986600933596492,
-      "rewards/accuracy_reward": 0.51015625,
-      "rewards/format_reward": 0.14765625,
-      "rewards/reflection_reward_pos": 0.00234375,
+      "loss": 0.002,
+      "reward": 0.6484375,
+      "reward_std": 0.09501747125759721,
+      "rewards/accuracy_reward": 0.64765625,
+      "rewards/reflection_reward_pos": 0.00078125,
       "step": 90
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 558.94765625,
+      "completion_length": 646.7421875,
       "epoch": 1.6311300639658848,
-      "grad_norm": 0.27353477478027344,
-      "kl": 0.246923828125,
+      "grad_norm": 0.09559585154056549,
+      "kl": 0.0030200958251953127,
       "learning_rate": 1.5302055798981605e-06,
-      "loss": 0.0191,
-      "reward": 0.6890625,
-      "reward_std": 0.22539028134196998,
-      "rewards/accuracy_reward": 0.51484375,
-      "rewards/format_reward": 0.171875,
-      "rewards/reflection_reward_pos": 0.00234375,
+      "loss": 0.0044,
+      "reward": 0.65625,
+      "reward_std": 0.11048543220385909,
+      "rewards/accuracy_reward": 0.65625,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 95
     },
     {
       "epoch": 1.716417910447761,
-      "grad_norm": 0.6983962059020996,
+      "grad_norm": 0.06905966252088547,
       "learning_rate": 1.3793001469249112e-06,
-      "loss": 0.0255,
+      "loss": 0.0084,
       "step": 100
     },
     {
       "epoch": 1.716417910447761,
       "eval_clip_ratio": 0.0,
-      "eval_completion_length": 535.5470247603834,
-      "eval_kl": 0.17427428614217252,
-      "eval_loss": 0.01942022331058979,
-      "eval_reward": 0.6449680511182109,
-      "eval_reward_std": 0.24596245542835124,
-      "eval_rewards/accuracy_reward": 0.461461661341853,
-      "eval_rewards/format_reward": 0.1814097444089457,
-      "eval_rewards/reflection_reward_pos": 0.0020966453674121405,
-      "eval_runtime": 5724.0277,
-      "eval_samples_per_second": 0.874,
-      "eval_steps_per_second": 0.027,
+      "eval_completion_length": 625.3352635782747,
+      "eval_kl": 0.0029512838052865417,
+      "eval_loss": 0.0050412570126354694,
+      "eval_reward": 0.6336861022364217,
+      "eval_reward_std": 0.12947621855872887,
+      "eval_rewards/accuracy_reward": 0.6328873801916933,
+      "eval_rewards/reflection_reward_pos": 0.0007987220447284345,
+      "eval_runtime": 6061.8369,
+      "eval_samples_per_second": 0.825,
+      "eval_steps_per_second": 0.026,
       "step": 100
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 561.4390625,
+      "completion_length": 643.7375,
       "epoch": 1.8017057569296375,
-      "grad_norm": 0.3275902569293976,
-      "kl": 0.16116943359375,
+      "grad_norm": 0.08424519002437592,
+      "kl": 0.0029918670654296873,
       "learning_rate": 1.2296174432791415e-06,
-      "loss": 0.0212,
-      "reward": 0.686328125,
-      "reward_std": 0.2524592101573944,
-      "rewards/accuracy_reward": 0.516015625,
-      "rewards/format_reward": 0.16796875,
-      "rewards/reflection_reward_pos": 0.00234375,
+      "loss": 0.0065,
+      "reward": 0.676171875,
+      "reward_std": 0.11103785866871477,
+      "rewards/accuracy_reward": 0.676171875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 105
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 557.31796875,
+      "completion_length": 644.7671875,
       "epoch": 1.886993603411514,
-      "grad_norm": 0.6603850722312927,
-      "kl": 0.16229248046875,
+      "grad_norm": 0.08362549543380737,
+      "kl": 0.00287628173828125,
       "learning_rate": 1.0826738041253211e-06,
-      "loss": 0.0244,
-      "reward": 0.64921875,
-      "reward_std": 0.22649513594806195,
-      "rewards/accuracy_reward": 0.484375,
-      "rewards/format_reward": 0.1625,
-      "rewards/reflection_reward_pos": 0.00234375,
+      "loss": 0.0102,
+      "reward": 0.6671875,
+      "reward_std": 0.12374368365854024,
+      "rewards/accuracy_reward": 0.6671875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 110
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 553.05,
+      "completion_length": 633.04375,
       "epoch": 1.9722814498933903,
-      "grad_norm": 0.3981146216392517,
-      "kl": 0.19359130859375,
+      "grad_norm": 0.0973709300160408,
+      "kl": 0.003045654296875,
       "learning_rate": 9.399578170010685e-07,
-      "loss": 0.0234,
-      "reward": 0.665625,
-      "reward_std": 0.2563262009993196,
-      "rewards/accuracy_reward": 0.49296875,
-      "rewards/format_reward": 0.16953125,
-      "rewards/reflection_reward_pos": 0.003125,
+      "loss": 0.0006,
+      "reward": 0.67265625,
+      "reward_std": 0.1182194116525352,
+      "rewards/accuracy_reward": 0.67265625,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 115
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 575.7098999023438,
+      "completion_length": 651.9445343017578,
       "epoch": 2.068230277185501,
-      "grad_norm": 0.39923951029777527,
-      "kl": 0.27755126953125,
+      "grad_norm": 0.07164692878723145,
+      "kl": 0.0029529571533203126,
       "learning_rate": 8.029152419343472e-07,
-      "loss": 0.016,
-      "reward": 0.67421875,
-      "reward_std": 0.21986600682139396,
-      "rewards/accuracy_reward": 0.49609375,
-      "rewards/format_reward": 0.17734375,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.0074,
+      "reward": 0.67265625,
+      "reward_std": 0.10496116001158953,
+      "rewards/accuracy_reward": 0.67265625,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 120
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 534.37890625,
+      "completion_length": 639.64375,
       "epoch": 2.1535181236673773,
-      "grad_norm": 0.7698010802268982,
-      "kl": 0.29091796875,
+      "grad_norm": 0.07949467748403549,
+      "kl": 0.0030269622802734375,
       "learning_rate": 6.729343654174626e-07,
-      "loss": 0.0192,
-      "reward": 0.67578125,
-      "reward_std": 0.24859222043305634,
-      "rewards/accuracy_reward": 0.4953125,
-      "rewards/format_reward": 0.17890625,
-      "rewards/reflection_reward_pos": 0.0015625,
+      "loss": 0.0081,
+      "reward": 0.6953125,
+      "reward_std": 0.11490484857931733,
+      "rewards/accuracy_reward": 0.6953125,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 125
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 565.246875,
+      "completion_length": 656.5984375,
       "epoch": 2.2388059701492535,
-      "grad_norm": 0.5925753116607666,
-      "kl": 0.27186279296875,
+      "grad_norm": 0.08217156678438187,
+      "kl": 0.0028797149658203124,
       "learning_rate": 5.513319366069343e-07,
-      "loss": 0.0179,
-      "reward": 0.66640625,
-      "reward_std": 0.24417280461639165,
-      "rewards/accuracy_reward": 0.51328125,
-      "rewards/format_reward": 0.15234375,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.0076,
+      "reward": 0.6890625,
+      "reward_std": 0.11711455835029483,
+      "rewards/accuracy_reward": 0.6890625,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 130
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 553.93984375,
+      "completion_length": 657.54296875,
       "epoch": 2.3240938166311302,
-      "grad_norm": 0.45327264070510864,
-      "kl": 0.29208984375,
+      "grad_norm": 0.08467955142259598,
+      "kl": 0.0028537750244140626,
       "learning_rate": 4.3933982822017883e-07,
-      "loss": 0.0209,
-      "reward": 0.6453125,
-      "reward_std": 0.20550290141254662,
-      "rewards/accuracy_reward": 0.46953125,
-      "rewards/format_reward": 0.17421875,
-      "rewards/reflection_reward_pos": 0.0015625,
+      "loss": 0.0046,
+      "reward": 0.67109375,
+      "reward_std": 0.10938057713210583,
+      "rewards/accuracy_reward": 0.6703125,
+      "rewards/reflection_reward_pos": 0.00078125,
       "step": 135
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 544.51875,
+      "completion_length": 642.66484375,
       "epoch": 2.4093816631130065,
-      "grad_norm": 11.81939697265625,
-      "kl": 0.615234375,
+      "grad_norm": 0.08187657594680786,
+      "kl": 0.0027801513671875,
       "learning_rate": 3.380925572585183e-07,
-      "loss": 0.0236,
-      "reward": 0.65546875,
-      "reward_std": 0.22649513762444257,
-      "rewards/accuracy_reward": 0.48359375,
-      "rewards/format_reward": 0.16953125,
-      "rewards/reflection_reward_pos": 0.00234375,
+      "loss": 0.008,
+      "reward": 0.6625,
+      "reward_std": 0.11711455713957548,
+      "rewards/accuracy_reward": 0.6625,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 140
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 531.19765625,
+      "completion_length": 622.1828125,
       "epoch": 2.4946695095948828,
-      "grad_norm": 0.4690285325050354,
-      "kl": 0.2049560546875,
+      "grad_norm": 0.08926448225975037,
+      "kl": 0.00284576416015625,
       "learning_rate": 2.4861579197570804e-07,
-      "loss": 0.0214,
-      "reward": 0.65546875,
-      "reward_std": 0.2375436789356172,
-      "rewards/accuracy_reward": 0.48359375,
-      "rewards/format_reward": 0.17109375,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.0096,
+      "reward": 0.678125,
+      "reward_std": 0.11711455807089806,
+      "rewards/accuracy_reward": 0.678125,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 145
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 557.59921875,
+      "completion_length": 636.60546875,
       "epoch": 2.579957356076759,
-      "grad_norm": 0.4469051957130432,
-      "kl": 0.2426513671875,
+      "grad_norm": 0.09873559325933456,
+      "kl": 0.00290679931640625,
       "learning_rate": 1.718159615201853e-07,
-      "loss": 0.0271,
-      "reward": 0.62734375,
-      "reward_std": 0.2264951358549297,
-      "rewards/accuracy_reward": 0.4875,
-      "rewards/format_reward": 0.13828125,
-      "rewards/reflection_reward_pos": 0.0015625,
+      "loss": 0.0088,
+      "reward": 0.66328125,
+      "reward_std": 0.12263882830739022,
+      "rewards/accuracy_reward": 0.6625,
+      "rewards/reflection_reward_pos": 0.00078125,
       "step": 150
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 544.9953125,
+      "completion_length": 641.1921875,
       "epoch": 2.6652452025586353,
-      "grad_norm": 0.5272353887557983,
-      "kl": 0.239892578125,
+      "grad_norm": 0.08141663670539856,
+      "kl": 0.00279998779296875,
       "learning_rate": 1.0847107350878571e-07,
-      "loss": 0.0235,
-      "reward": 0.68046875,
-      "reward_std": 0.23533396869897844,
-      "rewards/accuracy_reward": 0.5125,
-      "rewards/format_reward": 0.1671875,
+      "loss": 0.0056,
+      "reward": 0.66875,
+      "reward_std": 0.11269514048472047,
+      "rewards/accuracy_reward": 0.66796875,
       "rewards/reflection_reward_pos": 0.00078125,
       "step": 155
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 545.94296875,
+      "completion_length": 635.07578125,
       "epoch": 2.750533049040512,
-      "grad_norm": 0.6531479358673096,
-      "kl": 0.25457763671875,
+      "grad_norm": 0.07290147989988327,
+      "kl": 0.002706146240234375,
       "learning_rate": 5.922283255294164e-08,
-      "loss": 0.0213,
-      "reward": 0.70234375,
-      "reward_std": 0.26847960213199257,
-      "rewards/accuracy_reward": 0.52421875,
-      "rewards/format_reward": 0.17734375,
-      "rewards/reflection_reward_pos": 0.00078125,
+      "loss": 0.0028,
+      "reward": 0.7171875,
+      "reward_std": 0.09722718009725213,
+      "rewards/accuracy_reward": 0.7171875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 160
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 551.03359375,
+      "completion_length": 633.3,
       "epoch": 2.835820895522388,
-      "grad_norm": 0.6810430884361267,
-      "kl": 0.2883544921875,
+      "grad_norm": 0.07411785423755646,
+      "kl": 0.0027303695678710938,
       "learning_rate": 2.4570139579284723e-08,
-      "loss": 0.0245,
-      "reward": 0.671875,
-      "reward_std": 0.22097086254507303,
-      "rewards/accuracy_reward": 0.5046875,
-      "rewards/format_reward": 0.1640625,
-      "rewards/reflection_reward_pos": 0.003125,
+      "loss": 0.0062,
+      "reward": 0.6953125,
+      "reward_std": 0.10385630559176207,
+      "rewards/accuracy_reward": 0.6953125,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 165
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 562.71015625,
+      "completion_length": 639.90546875,
       "epoch": 2.9211087420042645,
-      "grad_norm": 0.718024492263794,
-      "kl": 0.270947265625,
+      "grad_norm": 0.09646094590425491,
+      "kl": 0.0026676177978515623,
       "learning_rate": 4.864037798685106e-09,
-      "loss": 0.017,
-      "reward": 0.68359375,
-      "reward_std": 0.23754367623478173,
-      "rewards/accuracy_reward": 0.5265625,
-      "rewards/format_reward": 0.15390625,
-      "rewards/reflection_reward_pos": 0.003125,
+      "loss": 0.0018,
+      "reward": 0.69296875,
+      "reward_std": 0.10717086931690574,
+      "rewards/accuracy_reward": 0.69296875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 170
     },
     {
       "clip_ratio": 0.0,
-      "completion_length": 597.6367235183716,
+      "completion_length": 644.8330116271973,
       "epoch": 2.9893390191897655,
-      "kl": 0.2867584228515625,
-      "reward": 0.63671875,
-      "reward_std": 0.20992232125718147,
-      "rewards/accuracy_reward": 0.4658203125,
-      "rewards/format_reward": 0.16796875,
-      "rewards/reflection_reward_pos": 0.0029296875,
+      "kl": 0.0027103424072265625,
+      "reward": 0.669921875,
+      "reward_std": 0.11877183895558119,
+      "rewards/accuracy_reward": 0.669921875,
+      "rewards/reflection_reward_pos": 0.0,
       "step": 174,
       "total_flos": 0.0,
-      "train_loss": 0.019510615981001962,
-      "train_runtime": 40590.3738,
-      "train_samples_per_second": 0.554,
+      "train_loss": 0.005782647762212089,
+      "train_runtime": 41644.7787,
+      "train_samples_per_second": 0.54,
       "train_steps_per_second": 0.004
     }
   ],

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:495f3baedf05f2b9cc79184cd9e8d5eedc0d5c8072d38bfd1f4d015f1254096b
-size 8184
+oid sha256:4db7f6d6c128be56fa07ba7fc1b230400695276f9ea554cbfedcc1a931511184
+size 7800