{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.3189326556543837,
  "eval_steps": 500,
  "global_step": 502,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 630.0,
      "epoch": 0.0006353240152477764,
      "grad_norm": 3.4243216514587402,
      "kl": 0.0,
      "learning_rate": 3.3333333333333335e-07,
      "loss": 0.0,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 337.14288330078125,
      "epoch": 0.0012706480304955528,
      "grad_norm": 5.606422424316406,
      "kl": 0.0,
      "learning_rate": 6.666666666666667e-07,
      "loss": 0.0,
      "reward": 0.3428571820259094,
      "reward_std": 0.09759000688791275,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 517.2857666015625,
      "epoch": 0.0019059720457433292,
      "grad_norm": 3.686295986175537,
      "kl": 0.0004425048828125,
      "learning_rate": 1.0000000000000002e-06,
      "loss": 0.0,
      "reward": 0.26031747460365295,
      "reward_std": 0.19519509375095367,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0317460335791111,
      "rewards/format_reward": 0.5714285969734192,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 638.4285888671875,
      "epoch": 0.0025412960609911056,
      "grad_norm": 3.40989089012146,
      "kl": 0.0004863739013671875,
      "learning_rate": 1.3333333333333334e-06,
      "loss": 0.0,
      "reward": 0.1714285910129547,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.4285714626312256,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.4285714626312256,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 376.4285888671875,
      "epoch": 0.0031766200762388818,
      "grad_norm": 4.474236011505127,
      "kl": 0.0008697509765625,
      "learning_rate": 1.6666666666666667e-06,
      "loss": 0.0,
      "reward": 0.40000003576278687,
      "reward_std": 0.40000003576278687,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.7142857313156128,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 484.71429443359375,
      "epoch": 0.0038119440914866584,
      "grad_norm": 4.529554843902588,
      "kl": 0.0179443359375,
      "learning_rate": 2.0000000000000003e-06,
      "loss": 0.0002,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 446.0000305175781,
      "epoch": 0.0044472681067344345,
      "grad_norm": 3.3631019592285156,
      "kl": 0.00128936767578125,
      "learning_rate": 2.3333333333333336e-06,
      "loss": 0.0,
      "reward": 0.8025974631309509,
      "reward_std": 0.4480050206184387,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.40259745717048645,
      "rewards/format_reward": 1.0,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 765.0000610351562,
      "epoch": 0.005082592121982211,
      "grad_norm": 3.205476999282837,
      "kl": 0.000675201416015625,
      "learning_rate": 2.666666666666667e-06,
      "loss": 0.0,
      "reward": 0.1714285910129547,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.4285714626312256,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.4285714626312256,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 628.5714721679688,
      "epoch": 0.005717916137229987,
      "grad_norm": 0.008113108575344086,
      "kl": 0.001678466796875,
      "learning_rate": 3e-06,
      "loss": 0.0003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 934.1428833007812,
      "epoch": 0.0063532401524777635,
      "grad_norm": 2.805612087249756,
      "kl": 0.00390625,
      "learning_rate": 3.3333333333333333e-06,
      "loss": 0.0,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 733.4285888671875,
      "epoch": 0.00698856416772554,
      "grad_norm": 0.014334071427583694,
      "kl": 0.0016326904296875,
      "learning_rate": 3.6666666666666666e-06,
      "loss": 0.0003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 545.5714721679688,
      "epoch": 0.007623888182973317,
      "grad_norm": 3.0349507331848145,
      "kl": 0.003936767578125,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 539.7142944335938,
      "epoch": 0.008259212198221092,
      "grad_norm": 2.7579803466796875,
      "kl": 0.00787353515625,
      "learning_rate": 4.333333333333334e-06,
      "loss": 0.0001,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 502.5714416503906,
      "epoch": 0.008894536213468869,
      "grad_norm": 0.06331096589565277,
      "kl": 0.0101318359375,
      "learning_rate": 4.666666666666667e-06,
      "loss": 0.0004,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 583.5714721679688,
      "epoch": 0.009529860228716646,
      "grad_norm": 1.8358005285263062,
      "kl": 0.0576171875,
      "learning_rate": 5e-06,
      "loss": 0.0009,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 528.4285888671875,
      "epoch": 0.010165184243964422,
      "grad_norm": 5.376899242401123,
      "kl": 0.173828125,
      "learning_rate": 4.999952797253148e-06,
      "loss": 0.0017,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.5714416503906,
      "epoch": 0.010800508259212199,
      "grad_norm": 5.1895365715026855,
      "kl": 0.1767578125,
      "learning_rate": 4.9998111909931225e-06,
      "loss": 0.0018,
      "reward": 0.5571428537368774,
      "reward_std": 0.12051477283239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.2142857313156128,
      "rewards/format_reward": 0.8571429252624512,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 449.5714416503906,
      "epoch": 0.011435832274459974,
      "grad_norm": 3.5275230407714844,
      "kl": 0.05126953125,
      "learning_rate": 4.999575187161439e-06,
      "loss": 0.0005,
      "reward": 0.6000000238418579,
      "reward_std": 0.432049423456192,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 0.8571429252624512,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 523.1428833007812,
      "epoch": 0.01207115628970775,
      "grad_norm": 3.147331953048706,
      "kl": 0.04541015625,
      "learning_rate": 4.9992447956603455e-06,
      "loss": 0.0005,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 330.14288330078125,
      "epoch": 0.012706480304955527,
      "grad_norm": 108.5281753540039,
      "kl": 3.046875,
      "learning_rate": 4.998820030352409e-06,
      "loss": 0.0305,
      "reward": 0.6884711980819702,
      "reward_std": 0.4107622504234314,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.31704264879226685,
      "rewards/format_reward": 1.0,
      "step": 20
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 359.0000305175781,
      "epoch": 0.013341804320203304,
      "grad_norm": 5.423805236816406,
      "kl": 0.1591796875,
      "learning_rate": 4.998300909059929e-06,
      "loss": 0.0016,
      "reward": 0.5269841551780701,
      "reward_std": 0.1777612417936325,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1269841343164444,
      "rewards/format_reward": 1.0,
      "step": 21
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 491.2857360839844,
      "epoch": 0.01397712833545108,
      "grad_norm": 1545.3717041015625,
      "kl": 23.0,
      "learning_rate": 4.997687453564198e-06,
      "loss": 0.2309,
      "reward": 0.5428571701049805,
      "reward_std": 0.6078847646713257,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 0.7142857313156128,
      "step": 22
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 503.5714416503906,
      "epoch": 0.014612452350698857,
      "grad_norm": 8.096309661865234,
      "kl": 0.69140625,
      "learning_rate": 4.9969796896045775e-06,
      "loss": 0.0069,
      "reward": 0.3142857253551483,
      "reward_std": 0.10690449923276901,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 23
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 604.1428833007812,
      "epoch": 0.015247776365946633,
      "grad_norm": 45758.625,
      "kl": 608.0,
      "learning_rate": 4.996177646877426e-06,
      "loss": 6.0704,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 24
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 605.1428833007812,
      "epoch": 0.01588310038119441,
      "grad_norm": 3.5074126720428467,
      "kl": 0.1083984375,
      "learning_rate": 4.995281359034851e-06,
      "loss": 0.0011,
      "reward": 0.05714286118745804,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.1428571492433548,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.1428571492433548,
      "step": 25
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.2857360839844,
      "epoch": 0.016518424396442185,
      "grad_norm": 9.928363800048828,
      "kl": 0.421875,
      "learning_rate": 4.994290863683296e-06,
      "loss": 0.0042,
      "reward": 0.11428572237491608,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.2857142984867096,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.2857142984867096,
      "step": 26
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 420.8571472167969,
      "epoch": 0.017153748411689963,
      "grad_norm": 30.34365463256836,
      "kl": 1.2890625,
      "learning_rate": 4.99320620238196e-06,
      "loss": 0.0129,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 27
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 618.7142944335938,
      "epoch": 0.017789072426937738,
      "grad_norm": 3.0826454162597656,
      "kl": 0.051513671875,
      "learning_rate": 4.99202742064106e-06,
      "loss": 0.0005,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 28
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 582.5714721679688,
      "epoch": 0.018424396442185513,
      "grad_norm": 3.330746650695801,
      "kl": 0.07666015625,
      "learning_rate": 4.990754567919917e-06,
      "loss": 0.0008,
      "reward": 0.4571428894996643,
      "reward_std": 0.35989415645599365,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.1714285910129547,
      "rewards/format_reward": 0.7142857313156128,
      "step": 29
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 440.2857360839844,
      "epoch": 0.01905972045743329,
      "grad_norm": 3.658856153488159,
      "kl": 0.1318359375,
      "learning_rate": 4.989387697624881e-06,
      "loss": 0.0013,
      "reward": 0.3086913228034973,
      "reward_std": 0.20198491215705872,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.02297702245414257,
      "rewards/format_reward": 0.7142857313156128,
      "step": 30
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 764.0000610351562,
      "epoch": 0.019695044472681066,
      "grad_norm": 2.7840328216552734,
      "kl": 0.078125,
      "learning_rate": 4.987926867107095e-06,
      "loss": 0.0008,
      "reward": 0.31462588906288147,
      "reward_std": 0.19902175664901733,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.028911566361784935,
      "rewards/format_reward": 0.7142857313156128,
      "step": 31
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 573.4285888671875,
      "epoch": 0.020330368487928845,
      "grad_norm": 3.031372308731079,
      "kl": 0.078125,
      "learning_rate": 4.986372137660078e-06,
      "loss": 0.0008,
      "reward": 0.485714316368103,
      "reward_std": 0.22677868604660034,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 32
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 460.14288330078125,
      "epoch": 0.02096569250317662,
      "grad_norm": 3.5374386310577393,
      "kl": 0.107421875,
      "learning_rate": 4.984723574517165e-06,
      "loss": 0.0011,
      "reward": 0.3362637758255005,
      "reward_std": 0.17483238875865936,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.021978024393320084,
      "rewards/format_reward": 0.8571429252624512,
      "step": 33
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 510.71429443359375,
      "epoch": 0.021601016518424398,
      "grad_norm": 3.355870246887207,
      "kl": 0.20703125,
      "learning_rate": 4.9829812468487655e-06,
      "loss": 0.0021,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 34
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 617.7142944335938,
      "epoch": 0.022236340533672173,
      "grad_norm": 2.325124979019165,
      "kl": 0.0849609375,
      "learning_rate": 4.981145227759457e-06,
      "loss": 0.0008,
      "reward": 0.5142857432365417,
      "reward_std": 0.39761197566986084,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 35
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 329.4285888671875,
      "epoch": 0.022871664548919948,
      "grad_norm": 4.9638495445251465,
      "kl": 0.224609375,
      "learning_rate": 4.979215594284924e-06,
      "loss": 0.0022,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 36
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 734.5714721679688,
      "epoch": 0.023506988564167726,
      "grad_norm": 0.06015148386359215,
      "kl": 0.08154296875,
      "learning_rate": 4.977192427388722e-06,
      "loss": 0.0011,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 37
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 481.5714416503906,
      "epoch": 0.0241423125794155,
      "grad_norm": 0.20115888118743896,
      "kl": 0.12109375,
      "learning_rate": 4.9750758119588824e-06,
      "loss": 0.0015,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 38
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 817.0000610351562,
      "epoch": 0.02477763659466328,
      "grad_norm": 2.3490183353424072,
      "kl": 0.0888671875,
      "learning_rate": 4.972865836804349e-06,
      "loss": 0.0009,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 39
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 622.1428833007812,
      "epoch": 0.025412960609911054,
      "grad_norm": 40.009159088134766,
      "kl": 2.34375,
      "learning_rate": 4.970562594651254e-06,
      "loss": 0.0234,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 40
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 498.71429443359375,
      "epoch": 0.026048284625158832,
      "grad_norm": 3.2044177055358887,
      "kl": 0.12109375,
      "learning_rate": 4.968166182139026e-06,
      "loss": 0.0012,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 41
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 480.4285888671875,
      "epoch": 0.026683608640406607,
      "grad_norm": 0.08217895030975342,
      "kl": 0.1484375,
      "learning_rate": 4.9656766998163306e-06,
      "loss": 0.0018,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 42
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 699.5714721679688,
      "epoch": 0.027318932655654382,
      "grad_norm": 3.2091763019561768,
      "kl": 0.0966796875,
      "learning_rate": 4.963094252136865e-06,
      "loss": 0.001,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 43
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 595.857177734375,
      "epoch": 0.02795425667090216,
      "grad_norm": 0.07719128578901291,
      "kl": 0.115234375,
      "learning_rate": 4.960418947454958e-06,
      "loss": 0.0014,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 44
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 598.0,
      "epoch": 0.028589580686149935,
      "grad_norm": 0.10212292522192001,
      "kl": 0.1298828125,
      "learning_rate": 4.957650898021038e-06,
      "loss": 0.0016,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 45
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 517.857177734375,
      "epoch": 0.029224904701397714,
      "grad_norm": 2.6977856159210205,
      "kl": 0.1337890625,
      "learning_rate": 4.954790219976915e-06,
      "loss": 0.0013,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 46
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 699.5714721679688,
      "epoch": 0.02986022871664549,
      "grad_norm": 0.09081219881772995,
      "kl": 0.10888671875,
      "learning_rate": 4.95183703335091e-06,
      "loss": 0.0014,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 47
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 375.5714416503906,
      "epoch": 0.030495552731893267,
      "grad_norm": 3.1593081951141357,
      "kl": 0.162109375,
      "learning_rate": 4.948791462052819e-06,
      "loss": 0.0016,
      "reward": 0.9428572654724121,
      "reward_std": 0.5740416646003723,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 48
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 605.5714721679688,
      "epoch": 0.031130876747141042,
      "grad_norm": 0.2238318771123886,
      "kl": 0.1396484375,
      "learning_rate": 4.945653633868716e-06,
      "loss": 0.0017,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 49
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 519.0,
      "epoch": 0.03176620076238882,
      "grad_norm": 3.9522361755371094,
      "kl": 0.1796875,
      "learning_rate": 4.942423680455584e-06,
      "loss": 0.0018,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 50
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 577.0,
      "epoch": 0.032401524777636595,
      "grad_norm": 3.095000743865967,
      "kl": 0.15234375,
      "learning_rate": 4.939101737335802e-06,
      "loss": 0.0015,
      "reward": 0.11428572237491608,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.2857142984867096,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.2857142984867096,
      "step": 51
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 651.2857666015625,
      "epoch": 0.03303684879288437,
      "grad_norm": 2.880178451538086,
      "kl": 0.14453125,
      "learning_rate": 4.935687943891447e-06,
      "loss": 0.0015,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 52
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 570.4285888671875,
      "epoch": 0.033672172808132145,
      "grad_norm": 6.103281021118164,
      "kl": 0.421875,
      "learning_rate": 4.932182443358458e-06,
      "loss": 0.0042,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 53
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 461.857177734375,
      "epoch": 0.03430749682337993,
      "grad_norm": 5.561570644378662,
      "kl": 0.283203125,
      "learning_rate": 4.928585382820616e-06,
      "loss": 0.0028,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 54
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 710.0000610351562,
      "epoch": 0.0349428208386277,
      "grad_norm": 0.06716505438089371,
      "kl": 0.1875,
      "learning_rate": 4.924896913203376e-06,
      "loss": 0.0022,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 55
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 725.5714721679688,
      "epoch": 0.035578144853875476,
      "grad_norm": 4.731691837310791,
      "kl": 0.130859375,
      "learning_rate": 4.921117189267535e-06,
      "loss": 0.0013,
      "reward": 0.3588235676288605,
      "reward_std": 0.43374723196029663,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.13025210797786713,
      "rewards/format_reward": 0.5714285969734192,
      "step": 56
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 665.2857666015625,
      "epoch": 0.03621346886912325,
      "grad_norm": 2.7444961071014404,
      "kl": 0.18359375,
      "learning_rate": 4.917246369602742e-06,
      "loss": 0.0018,
      "reward": 0.4285714626312256,
      "reward_std": 0.4680252969264984,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.7142857313156128,
      "step": 57
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 648.0,
      "epoch": 0.036848792884371026,
      "grad_norm": 3.6189987659454346,
      "kl": 0.20703125,
      "learning_rate": 4.9132846166208355e-06,
      "loss": 0.0021,
      "reward": 0.5142857432365417,
      "reward_std": 0.6309479475021362,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 0.5714285969734192,
      "step": 58
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 483.14288330078125,
      "epoch": 0.03748411689961881,
      "grad_norm": 0.15579625964164734,
      "kl": 0.26953125,
      "learning_rate": 4.9092320965490365e-06,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 59
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.857177734375,
      "epoch": 0.03811944091486658,
      "grad_norm": 3.2243692874908447,
      "kl": 0.154296875,
      "learning_rate": 4.905088979422971e-06,
      "loss": 0.0015,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 60
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 446.0000305175781,
      "epoch": 0.03875476493011436,
      "grad_norm": 3.187040090560913,
      "kl": 0.2392578125,
      "learning_rate": 4.900855439079536e-06,
      "loss": 0.0024,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 61
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 619.7142944335938,
      "epoch": 0.03939008894536213,
      "grad_norm": 3.501086473464966,
      "kl": 0.205078125,
      "learning_rate": 4.8965316531496055e-06,
      "loss": 0.0021,
      "reward": 0.3428571820259094,
      "reward_std": 0.09759001433849335,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 62
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 598.0,
      "epoch": 0.040025412960609914,
      "grad_norm": 4.5344557762146,
      "kl": 0.1484375,
      "learning_rate": 4.892117803050578e-06,
      "loss": 0.0015,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 63
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 507.0000305175781,
      "epoch": 0.04066073697585769,
      "grad_norm": 6.242559432983398,
      "kl": 0.392578125,
      "learning_rate": 4.887614073978761e-06,
      "loss": 0.0039,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 64
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 757.1428833007812,
      "epoch": 0.041296060991105464,
      "grad_norm": 10.64748764038086,
      "kl": 0.53515625,
      "learning_rate": 4.883020654901609e-06,
      "loss": 0.0054,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 65
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 585.4285888671875,
      "epoch": 0.04193138500635324,
      "grad_norm": 3.1377880573272705,
      "kl": 0.2119140625,
      "learning_rate": 4.878337738549785e-06,
      "loss": 0.0021,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 66
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 604.2857666015625,
      "epoch": 0.042566709021601014,
      "grad_norm": 0.17088104784488678,
      "kl": 0.2490234375,
      "learning_rate": 4.873565521409082e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 67
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 842.5714721679688,
      "epoch": 0.043202033036848796,
      "grad_norm": 3.577969551086426,
      "kl": 0.384765625,
      "learning_rate": 4.868704203712173e-06,
      "loss": 0.0038,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 68
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 841.0000610351562,
      "epoch": 0.04383735705209657,
      "grad_norm": 3.18711256980896,
      "kl": 0.2080078125,
      "learning_rate": 4.86375398943021e-06,
      "loss": 0.0021,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 69
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 525.4285888671875,
      "epoch": 0.044472681067344345,
      "grad_norm": 2.3753271102905273,
      "kl": 0.177734375,
      "learning_rate": 4.858715086264274e-06,
      "loss": 0.0018,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 70
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 406.8571472167969,
      "epoch": 0.04510800508259212,
      "grad_norm": 3.5183515548706055,
      "kl": 0.2138671875,
      "learning_rate": 4.853587705636646e-06,
      "loss": 0.0021,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 71
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 630.5714721679688,
      "epoch": 0.045743329097839895,
      "grad_norm": 2.6844465732574463,
      "kl": 0.103515625,
      "learning_rate": 4.84837206268195e-06,
      "loss": 0.001,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 72
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 464.14288330078125,
      "epoch": 0.04637865311308768,
      "grad_norm": 3.63313627243042,
      "kl": 0.1591796875,
      "learning_rate": 4.8430683762381195e-06,
      "loss": 0.0016,
      "reward": 0.2571428716182709,
      "reward_std": 0.19023796916007996,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 73
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 507.2857360839844,
      "epoch": 0.04701397712833545,
      "grad_norm": 2.707540988922119,
      "kl": 0.111328125,
      "learning_rate": 4.837676868837213e-06,
      "loss": 0.0011,
      "reward": 0.4023166298866272,
      "reward_std": 0.004252949263900518,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0023166025057435036,
      "rewards/format_reward": 1.0,
      "step": 74
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 558.857177734375,
      "epoch": 0.04764930114358323,
      "grad_norm": 3.0911412239074707,
      "kl": 0.1474609375,
      "learning_rate": 4.832197766696085e-06,
      "loss": 0.0015,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 75
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 460.4285888671875,
      "epoch": 0.048284625158831,
      "grad_norm": 3.5635647773742676,
      "kl": 0.1357421875,
      "learning_rate": 4.826631299706887e-06,
      "loss": 0.0014,
      "reward": 0.5032258033752441,
      "reward_std": 0.2984292209148407,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.18894009292125702,
      "rewards/format_reward": 0.8571429252624512,
      "step": 76
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.2857360839844,
      "epoch": 0.04891994917407878,
      "grad_norm": 0.05187558755278587,
      "kl": 0.1611328125,
      "learning_rate": 4.820977701427424e-06,
      "loss": 0.0019,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 77
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.14288330078125,
      "epoch": 0.04955527318932656,
      "grad_norm": 0.04543861746788025,
      "kl": 0.130859375,
      "learning_rate": 4.81523720907136e-06,
      "loss": 0.0016,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 78
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 551.1428833007812,
      "epoch": 0.05019059720457433,
      "grad_norm": 0.45131492614746094,
      "kl": 0.28515625,
      "learning_rate": 4.809410063498254e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 79
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 379.5714416503906,
      "epoch": 0.05082592121982211,
      "grad_norm": 3.854992151260376,
      "kl": 0.1591796875,
      "learning_rate": 4.8034965092034656e-06,
      "loss": 0.0016,
      "reward": 1.2000001668930054,
      "reward_std": 0.3829708993434906,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.8571429252624512,
      "rewards/format_reward": 0.8571429252624512,
      "step": 80
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 566.857177734375,
      "epoch": 0.05146124523506988,
      "grad_norm": 41.30952453613281,
      "kl": 1.9921875,
      "learning_rate": 4.797496794307889e-06,
      "loss": 0.0199,
      "reward": 0.2489795982837677,
      "reward_std": 0.23831285536289215,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.020408164709806442,
      "rewards/format_reward": 0.5714285969734192,
      "step": 81
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 448.71429443359375,
      "epoch": 0.052096569250317665,
      "grad_norm": 4.601639270782471,
      "kl": 0.2412109375,
      "learning_rate": 4.791411170547545e-06,
      "loss": 0.0024,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 82
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 614.5714721679688,
      "epoch": 0.05273189326556544,
      "grad_norm": 4.209871768951416,
      "kl": 0.396484375,
      "learning_rate": 4.785239893263017e-06,
      "loss": 0.004,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 83
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 410.4285888671875,
      "epoch": 0.053367217280813214,
      "grad_norm": 4.049778938293457,
      "kl": 0.171875,
      "learning_rate": 4.778983221388742e-06,
      "loss": 0.0017,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 84
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 421.71429443359375,
      "epoch": 0.05400254129606099,
      "grad_norm": 5.517159461975098,
      "kl": 0.318359375,
      "learning_rate": 4.77264141744214e-06,
      "loss": 0.0032,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 85
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.8571472167969,
      "epoch": 0.054637865311308764,
      "grad_norm": 1.0127395391464233,
      "kl": 0.37109375,
      "learning_rate": 4.766214747512603e-06,
      "loss": 0.004,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 86
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 502.2857360839844,
      "epoch": 0.055273189326556546,
      "grad_norm": 5.1016316413879395,
      "kl": 0.625,
      "learning_rate": 4.759703481250331e-06,
      "loss": 0.0062,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 87
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.5714416503906,
      "epoch": 0.05590851334180432,
      "grad_norm": 3205.96337890625,
      "kl": 94.0,
      "learning_rate": 4.753107891855015e-06,
      "loss": 0.9386,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 88
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 162.0,
      "epoch": 0.056543837357052096,
      "grad_norm": 5.175131797790527,
      "kl": 0.349609375,
      "learning_rate": 4.746428256064375e-06,
      "loss": 0.0035,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 89
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 255.2857208251953,
      "epoch": 0.05717916137229987,
      "grad_norm": 4.350471496582031,
      "kl": 0.251953125,
      "learning_rate": 4.7396648541425534e-06,
      "loss": 0.0025,
      "reward": 0.9142858386039734,
      "reward_std": 0.6202918887138367,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 0.8571429252624512,
      "step": 90
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 515.7142944335938,
      "epoch": 0.05781448538754765,
      "grad_norm": 3.159823417663574,
      "kl": 0.1904296875,
      "learning_rate": 4.732817969868348e-06,
      "loss": 0.0019,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 91
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 504.5714416503906,
      "epoch": 0.05844980940279543,
      "grad_norm": 3.7896342277526855,
      "kl": 0.2353515625,
      "learning_rate": 4.7258878905233095e-06,
      "loss": 0.0023,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 92
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 355.5714416503906,
      "epoch": 0.0590851334180432,
      "grad_norm": 2.87099289894104,
      "kl": 0.1962890625,
      "learning_rate": 4.718874906879688e-06,
      "loss": 0.002,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645264148712,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 93
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 561.0,
      "epoch": 0.05972045743329098,
      "grad_norm": 3.082097053527832,
      "kl": 0.220703125,
      "learning_rate": 4.711779313188231e-06,
      "loss": 0.0022,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 94
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.0000305175781,
      "epoch": 0.06035578144853875,
      "grad_norm": 0.06256424635648727,
      "kl": 0.2138671875,
      "learning_rate": 4.70460140716584e-06,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 95
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 695.1428833007812,
      "epoch": 0.060991105463786534,
      "grad_norm": 3.0291473865509033,
      "kl": 0.412109375,
      "learning_rate": 4.697341489983076e-06,
      "loss": 0.0041,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 96
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 459.4285888671875,
      "epoch": 0.06162642947903431,
      "grad_norm": 3.461120843887329,
      "kl": 0.291015625,
      "learning_rate": 4.6899998662515215e-06,
      "loss": 0.0029,
      "reward": 0.4714285731315613,
      "reward_std": 0.06681530177593231,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0714285746216774,
      "rewards/format_reward": 1.0,
      "step": 97
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 626.4285888671875,
      "epoch": 0.062261753494282084,
      "grad_norm": 0.07474807649850845,
      "kl": 0.3203125,
      "learning_rate": 4.682576844011007e-06,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 98
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 594.2857666015625,
      "epoch": 0.06289707750952986,
      "grad_norm": 0.3679976463317871,
      "kl": 0.28515625,
      "learning_rate": 4.675072734716678e-06,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 99
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 627.1428833007812,
      "epoch": 0.06353240152477764,
      "grad_norm": 39.379425048828125,
      "kl": 1.5390625,
      "learning_rate": 4.667487853225931e-06,
      "loss": 0.0153,
      "reward": 0.485714316368103,
      "reward_std": 0.42983949184417725,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 100
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 636.0,
      "epoch": 0.06416772554002541,
      "grad_norm": 0.5123635530471802,
      "kl": 0.1962890625,
      "learning_rate": 4.659822517785203e-06,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 101
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 623.7142944335938,
      "epoch": 0.06480304955527319,
      "grad_norm": 2.7022430896759033,
      "kl": 0.2314453125,
      "learning_rate": 4.6520770500166165e-06,
      "loss": 0.0023,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 102
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 338.4285888671875,
      "epoch": 0.06543837357052097,
      "grad_norm": 3.6665234565734863,
      "kl": 0.294921875,
      "learning_rate": 4.644251774904487e-06,
      "loss": 0.0029,
      "reward": 1.057142972946167,
      "reward_std": 0.5968170166015625,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.7142857313156128,
      "rewards/format_reward": 0.8571429252624512,
      "step": 103
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 563.5714721679688,
      "epoch": 0.06607369758576874,
      "grad_norm": 0.09322147816419601,
      "kl": 0.23828125,
      "learning_rate": 4.636347020781684e-06,
      "loss": 0.0027,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 104
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 734.1428833007812,
      "epoch": 0.06670902160101652,
      "grad_norm": 2.2183244228363037,
      "kl": 0.177734375,
      "learning_rate": 4.6283631193158605e-06,
      "loss": 0.0018,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 105
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 495.4285888671875,
      "epoch": 0.06734434561626429,
      "grad_norm": 3.992103338241577,
      "kl": 0.345703125,
      "learning_rate": 4.620300405495532e-06,
      "loss": 0.0035,
      "reward": 0.2571428716182709,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.4285714626312256,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 106
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 449.4285888671875,
      "epoch": 0.06797966963151207,
      "grad_norm": 4.281430244445801,
      "kl": 0.26953125,
      "learning_rate": 4.612159217616022e-06,
      "loss": 0.0027,
      "reward": 0.2571428716182709,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.4285714626312256,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 107
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 463.2857360839844,
      "epoch": 0.06861499364675985,
      "grad_norm": 3.0798134803771973,
      "kl": 0.2158203125,
      "learning_rate": 4.603939897265268e-06,
      "loss": 0.0022,
      "reward": 0.5067504644393921,
      "reward_std": 0.28243502974510193,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.10675039887428284,
      "rewards/format_reward": 1.0,
      "step": 108
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 534.7142944335938,
      "epoch": 0.06925031766200762,
      "grad_norm": 3.179516077041626,
      "kl": 0.236328125,
      "learning_rate": 4.595642789309492e-06,
      "loss": 0.0024,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 109
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 673.1428833007812,
      "epoch": 0.0698856416772554,
      "grad_norm": 2.751520872116089,
      "kl": 0.228515625,
      "learning_rate": 4.587268241878724e-06,
      "loss": 0.0023,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 110
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 506.71429443359375,
      "epoch": 0.07052096569250317,
      "grad_norm": 0.06256023049354553,
      "kl": 0.2490234375,
      "learning_rate": 4.578816606352205e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 111
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 705.5714721679688,
      "epoch": 0.07115628970775095,
      "grad_norm": 0.08651240170001984,
      "kl": 0.1826171875,
      "learning_rate": 4.570288237343632e-06,
      "loss": 0.0021,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 112
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 667.2857666015625,
      "epoch": 0.07179161372299873,
      "grad_norm": 1.87228262424469,
      "kl": 0.1552734375,
      "learning_rate": 4.561683492686289e-06,
      "loss": 0.0016,
      "reward": 0.37142857909202576,
      "reward_std": 0.335232675075531,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.5714285969734192,
      "step": 113
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 511.857177734375,
      "epoch": 0.0724269377382465,
      "grad_norm": 0.06886231899261475,
      "kl": 0.21875,
      "learning_rate": 4.5530027334180285e-06,
      "loss": 0.0025,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 114
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 746.857177734375,
      "epoch": 0.07306226175349428,
      "grad_norm": 2.801109552383423,
      "kl": 0.2060546875,
      "learning_rate": 4.544246323766122e-06,
      "loss": 0.0021,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 115
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 639.5714721679688,
      "epoch": 0.07369758576874205,
      "grad_norm": 2.9071028232574463,
      "kl": 0.2373046875,
      "learning_rate": 4.535414631131983e-06,
      "loss": 0.0024,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 116
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 760.2857666015625,
      "epoch": 0.07433290978398983,
      "grad_norm": 2.4393320083618164,
      "kl": 0.201171875,
      "learning_rate": 4.526508026075746e-06,
      "loss": 0.002,
      "reward": 0.2571428716182709,
      "reward_std": 0.19023796916007996,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 117
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 729.7142944335938,
      "epoch": 0.07496823379923762,
      "grad_norm": 7.918629169464111,
      "kl": 0.671875,
      "learning_rate": 4.517526882300721e-06,
      "loss": 0.0067,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 118
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 745.1428833007812,
      "epoch": 0.07560355781448538,
      "grad_norm": 2.229508876800537,
      "kl": 0.185546875,
      "learning_rate": 4.508471576637713e-06,
      "loss": 0.0019,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 119
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 508.4285888671875,
      "epoch": 0.07623888182973317,
      "grad_norm": 2.4816155433654785,
      "kl": 0.1845703125,
      "learning_rate": 4.499342489029211e-06,
      "loss": 0.0018,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 120
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 535.857177734375,
      "epoch": 0.07687420584498093,
      "grad_norm": 3.3153207302093506,
      "kl": 0.2490234375,
      "learning_rate": 4.490140002513449e-06,
      "loss": 0.0025,
      "reward": 0.43708792328834534,
      "reward_std": 0.0981253907084465,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.03708791360259056,
      "rewards/format_reward": 1.0,
      "step": 121
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 776.1428833007812,
      "epoch": 0.07750952986022872,
      "grad_norm": 2.2401111125946045,
      "kl": 0.1484375,
      "learning_rate": 4.48086450320833e-06,
      "loss": 0.0015,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 122
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 764.7142944335938,
      "epoch": 0.0781448538754765,
      "grad_norm": 2.0424013137817383,
      "kl": 0.1806640625,
      "learning_rate": 4.4715163802952266e-06,
      "loss": 0.0018,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 123
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 543.7142944335938,
      "epoch": 0.07878017789072427,
      "grad_norm": 0.1234215497970581,
      "kl": 0.25390625,
      "learning_rate": 4.462096026002655e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 124
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 741.5714721679688,
      "epoch": 0.07941550190597205,
      "grad_norm": 2.020581007003784,
      "kl": 0.1884765625,
      "learning_rate": 4.4526038355898144e-06,
      "loss": 0.0019,
      "reward": 0.36326533555984497,
      "reward_std": 0.16554531455039978,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.020408164709806442,
      "rewards/format_reward": 0.8571429252624512,
      "step": 125
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 593.857177734375,
      "epoch": 0.08005082592121983,
      "grad_norm": 2.3378539085388184,
      "kl": 0.2373046875,
      "learning_rate": 4.4430402073300035e-06,
      "loss": 0.0024,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 126
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.5714721679688,
      "epoch": 0.0806861499364676,
      "grad_norm": 0.06773251295089722,
      "kl": 0.234375,
      "learning_rate": 4.433405542493909e-06,
      "loss": 0.0026,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 127
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 703.1428833007812,
      "epoch": 0.08132147395171538,
      "grad_norm": 3.0085108280181885,
      "kl": 0.1630859375,
      "learning_rate": 4.4237002453327734e-06,
      "loss": 0.0016,
      "reward": 0.41020408272743225,
      "reward_std": 0.026997461915016174,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.010204082354903221,
      "rewards/format_reward": 1.0,
      "step": 128
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.2857360839844,
      "epoch": 0.08195679796696315,
      "grad_norm": 3.266486644744873,
      "kl": 0.171875,
      "learning_rate": 4.4139247230614245e-06,
      "loss": 0.0017,
      "reward": 0.4520833492279053,
      "reward_std": 0.12870661914348602,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0520833320915699,
      "rewards/format_reward": 1.0,
      "step": 129
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.2857360839844,
      "epoch": 0.08259212198221093,
      "grad_norm": 3.562208414077759,
      "kl": 0.322265625,
      "learning_rate": 4.404079385841201e-06,
      "loss": 0.0032,
      "reward": 0.4571428894996643,
      "reward_std": 0.2507132887840271,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 130
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 367.0000305175781,
      "epoch": 0.08322744599745871,
      "grad_norm": 0.06771758943796158,
      "kl": 0.244140625,
      "learning_rate": 4.394164646762734e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 131
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 523.857177734375,
      "epoch": 0.08386277001270648,
      "grad_norm": 2.9015965461730957,
      "kl": 0.220703125,
      "learning_rate": 4.384180921828618e-06,
      "loss": 0.0022,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 132
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 519.2857666015625,
      "epoch": 0.08449809402795426,
      "grad_norm": 3.0210537910461426,
      "kl": 0.1689453125,
      "learning_rate": 4.374128629935955e-06,
      "loss": 0.0017,
      "reward": 0.9833123087882996,
      "reward_std": 0.4177902638912201,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5833122730255127,
      "rewards/format_reward": 1.0,
      "step": 133
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 358.5714416503906,
      "epoch": 0.08513341804320203,
      "grad_norm": 2.964980363845825,
      "kl": 0.1962890625,
      "learning_rate": 4.364008192858781e-06,
      "loss": 0.002,
      "reward": 0.4098522365093231,
      "reward_std": 0.016825860366225243,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.009852217510342598,
      "rewards/format_reward": 1.0,
      "step": 134
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 473.5714416503906,
      "epoch": 0.08576874205844981,
      "grad_norm": 2.856916666030884,
      "kl": 0.2119140625,
      "learning_rate": 4.353820035230366e-06,
      "loss": 0.0021,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 135
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 507.2857360839844,
      "epoch": 0.08640406607369759,
      "grad_norm": 3.613862991333008,
      "kl": 0.169921875,
      "learning_rate": 4.3435645845254e-06,
      "loss": 0.0017,
      "reward": 0.6571429371833801,
      "reward_std": 0.5126960277557373,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 136
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 451.4285888671875,
      "epoch": 0.08703939008894536,
      "grad_norm": 0.05980609729886055,
      "kl": 0.177734375,
      "learning_rate": 4.333242271042054e-06,
      "loss": 0.0021,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 137
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 515.7142944335938,
      "epoch": 0.08767471410419314,
      "grad_norm": 3.2973406314849854,
      "kl": 0.248046875,
      "learning_rate": 4.32285352788393e-06,
      "loss": 0.0025,
      "reward": 0.5261905193328857,
      "reward_std": 0.28607451915740967,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1547619104385376,
      "rewards/format_reward": 1.0,
      "step": 138
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 384.14288330078125,
      "epoch": 0.08831003811944091,
      "grad_norm": 2.6424343585968018,
      "kl": 0.31640625,
      "learning_rate": 4.312398790941882e-06,
      "loss": 0.0032,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 139
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 326.8571472167969,
      "epoch": 0.08894536213468869,
      "grad_norm": 3.2031986713409424,
      "kl": 0.1884765625,
      "learning_rate": 4.301878498875735e-06,
      "loss": 0.0019,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 140
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 366.71429443359375,
      "epoch": 0.08958068614993647,
      "grad_norm": 0.29011043906211853,
      "kl": 0.232421875,
      "learning_rate": 4.291293093095873e-06,
      "loss": 0.0026,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 141
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 353.5714416503906,
      "epoch": 0.09021601016518424,
      "grad_norm": 0.05228522792458534,
      "kl": 0.1728515625,
      "learning_rate": 4.280643017744723e-06,
      "loss": 0.002,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 142
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 571.857177734375,
      "epoch": 0.09085133418043202,
      "grad_norm": 4.668937683105469,
      "kl": 0.255859375,
      "learning_rate": 4.269928719678117e-06,
      "loss": 0.0026,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 143
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 441.0000305175781,
      "epoch": 0.09148665819567979,
      "grad_norm": 3.7214486598968506,
      "kl": 0.42578125,
      "learning_rate": 4.2591506484465426e-06,
      "loss": 0.0042,
      "reward": 0.3428571820259094,
      "reward_std": 0.09759001433849335,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 144
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 493.14288330078125,
      "epoch": 0.09212198221092757,
      "grad_norm": 0.11755923926830292,
      "kl": 0.28515625,
      "learning_rate": 4.248309256276283e-06,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 145
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 365.4285888671875,
      "epoch": 0.09275730622617535,
      "grad_norm": 2.491922378540039,
      "kl": 0.2099609375,
      "learning_rate": 4.23740499805044e-06,
      "loss": 0.0021,
      "reward": 0.38035714626312256,
      "reward_std": 0.0817723423242569,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.008928571827709675,
      "rewards/format_reward": 1.0,
      "step": 146
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 363.4285888671875,
      "epoch": 0.09339263024142312,
      "grad_norm": 3.340977191925049,
      "kl": 0.185546875,
      "learning_rate": 4.22643833128985e-06,
      "loss": 0.0018,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 147
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 244.57144165039062,
      "epoch": 0.0940279542566709,
      "grad_norm": 0.05019477382302284,
      "kl": 0.2255859375,
      "learning_rate": 4.215409716133885e-06,
      "loss": 0.0035,
      "reward": 1.4000002145767212,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 1.0,
      "rewards/format_reward": 1.0,
      "step": 148
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 372.2857360839844,
      "epoch": 0.09466327827191867,
      "grad_norm": 3.1343963146209717,
      "kl": 0.34375,
      "learning_rate": 4.204319615321151e-06,
      "loss": 0.0034,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 149
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 355.5714416503906,
      "epoch": 0.09529860228716645,
      "grad_norm": 3.144193172454834,
      "kl": 0.35546875,
      "learning_rate": 4.193168494170065e-06,
      "loss": 0.0035,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 150
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.857177734375,
      "epoch": 0.09593392630241424,
      "grad_norm": 3.4582438468933105,
      "kl": 0.27734375,
      "learning_rate": 4.181956820559339e-06,
      "loss": 0.0028,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 151
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 556.857177734375,
      "epoch": 0.096569250317662,
      "grad_norm": 2.9669277667999268,
      "kl": 0.259765625,
      "learning_rate": 4.170685064908342e-06,
      "loss": 0.0026,
      "reward": 0.4870130121707916,
      "reward_std": 0.23021474480628967,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.08701299130916595,
      "rewards/format_reward": 1.0,
      "step": 152
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 561.1428833007812,
      "epoch": 0.09720457433290978,
      "grad_norm": 0.06397932022809982,
      "kl": 0.265625,
      "learning_rate": 4.159353700157365e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 153
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.8571472167969,
      "epoch": 0.09783989834815757,
      "grad_norm": 0.12080562859773636,
      "kl": 0.369140625,
      "learning_rate": 4.14796320174778e-06,
      "loss": 0.004,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 154
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 583.7142944335938,
      "epoch": 0.09847522236340533,
      "grad_norm": 0.06819948554039001,
      "kl": 0.298828125,
      "learning_rate": 4.136514047602087e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 155
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 347.14288330078125,
      "epoch": 0.09911054637865312,
      "grad_norm": 0.07698381692171097,
      "kl": 0.3984375,
      "learning_rate": 4.1250067181038635e-06,
      "loss": 0.0043,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 156
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 692.1428833007812,
      "epoch": 0.09974587039390088,
      "grad_norm": 0.17737269401550293,
      "kl": 0.28125,
      "learning_rate": 4.113441696077608e-06,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 157
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 341.5714416503906,
      "epoch": 0.10038119440914867,
      "grad_norm": 4.598702907562256,
      "kl": 0.4296875,
      "learning_rate": 4.101819466768484e-06,
      "loss": 0.0043,
      "reward": 0.8134453892707825,
      "reward_std": 0.18902148306369781,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4134454131126404,
      "rewards/format_reward": 1.0,
      "step": 158
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 455.2857360839844,
      "epoch": 0.10101651842439645,
      "grad_norm": 0.07675225287675858,
      "kl": 0.302734375,
      "learning_rate": 4.0901405178219535e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 159
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 752.2857666015625,
      "epoch": 0.10165184243964422,
      "grad_norm": 1.9616703987121582,
      "kl": 0.2080078125,
      "learning_rate": 4.078405339263326e-06,
      "loss": 0.0021,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 160
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 349.8571472167969,
      "epoch": 0.102287166454892,
      "grad_norm": 3.1100926399230957,
      "kl": 0.2451171875,
      "learning_rate": 4.06661442347719e-06,
      "loss": 0.0025,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 161
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 616.1428833007812,
      "epoch": 0.10292249047013977,
      "grad_norm": 2.4743812084198,
      "kl": 0.296875,
      "learning_rate": 4.054768265186758e-06,
      "loss": 0.003,
      "reward": 0.4129870533943176,
      "reward_std": 0.034360405057668686,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.012987013906240463,
      "rewards/format_reward": 1.0,
      "step": 162
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 359.4285888671875,
      "epoch": 0.10355781448538755,
      "grad_norm": 3.092137336730957,
      "kl": 0.3046875,
      "learning_rate": 4.0428673614331036e-06,
      "loss": 0.0031,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 163
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.5714416503906,
      "epoch": 0.10419313850063533,
      "grad_norm": 1.3710603713989258,
      "kl": 0.302734375,
      "learning_rate": 4.030912211554316e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 164
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 477.5714416503906,
      "epoch": 0.1048284625158831,
      "grad_norm": 2.647684097290039,
      "kl": 0.1826171875,
      "learning_rate": 4.018903317164539e-06,
      "loss": 0.0018,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 165
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 545.857177734375,
      "epoch": 0.10546378653113088,
      "grad_norm": 0.07831098884344101,
      "kl": 0.1650390625,
      "learning_rate": 4.006841182132932e-06,
      "loss": 0.0019,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 166
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 517.857177734375,
      "epoch": 0.10609911054637865,
      "grad_norm": 0.6324580311775208,
      "kl": 0.296875,
      "learning_rate": 3.9947263125625195e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 167
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 451.5714416503906,
      "epoch": 0.10673443456162643,
      "grad_norm": 2.877990961074829,
      "kl": 0.166015625,
      "learning_rate": 3.982559216768967e-06,
      "loss": 0.0017,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 168
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 527.2857666015625,
      "epoch": 0.10736975857687421,
      "grad_norm": 0.31927597522735596,
      "kl": 0.310546875,
      "learning_rate": 3.970340405259245e-06,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 169
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 545.7142944335938,
      "epoch": 0.10800508259212198,
      "grad_norm": 2.99507474899292,
      "kl": 0.28125,
      "learning_rate": 3.958070390710214e-06,
      "loss": 0.0028,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 170
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 430.71429443359375,
      "epoch": 0.10864040660736976,
      "grad_norm": 3.1845176219940186,
      "kl": 0.302734375,
      "learning_rate": 3.945749687947109e-06,
      "loss": 0.003,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 171
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 496.4285888671875,
      "epoch": 0.10927573062261753,
      "grad_norm": 2.674358606338501,
      "kl": 0.375,
      "learning_rate": 3.933378813921942e-06,
      "loss": 0.0037,
      "reward": 0.46666669845581055,
      "reward_std": 0.11547007411718369,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.06666667759418488,
      "rewards/format_reward": 1.0,
      "step": 172
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 722.1428833007812,
      "epoch": 0.10991105463786531,
      "grad_norm": 1.7992055416107178,
      "kl": 0.205078125,
      "learning_rate": 3.920958287691811e-06,
      "loss": 0.002,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 173
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 436.71429443359375,
      "epoch": 0.11054637865311309,
      "grad_norm": 0.03782209753990173,
      "kl": 0.1591796875,
      "learning_rate": 3.908488630397121e-06,
      "loss": 0.0019,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 174
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 446.14288330078125,
      "epoch": 0.11118170266836086,
      "grad_norm": 2.619063138961792,
      "kl": 0.1796875,
      "learning_rate": 3.8959703652397175e-06,
      "loss": 0.0018,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 175
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 705.5714721679688,
      "epoch": 0.11181702668360864,
      "grad_norm": 2.71985125541687,
      "kl": 0.267578125,
      "learning_rate": 3.883404017460935e-06,
      "loss": 0.0027,
      "reward": 0.2857142984867096,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 176
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 460.71429443359375,
      "epoch": 0.11245235069885642,
      "grad_norm": 3.090186595916748,
      "kl": 0.2109375,
      "learning_rate": 3.870790114319559e-06,
      "loss": 0.0021,
      "reward": 0.6571429371833801,
      "reward_std": 0.5126960277557373,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 177
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 482.71429443359375,
      "epoch": 0.11308767471410419,
      "grad_norm": 0.13239029049873352,
      "kl": 0.287109375,
      "learning_rate": 3.858129185069701e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 178
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 671.5714721679688,
      "epoch": 0.11372299872935197,
      "grad_norm": 2.1125059127807617,
      "kl": 0.27734375,
      "learning_rate": 3.845421760938597e-06,
      "loss": 0.0028,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 179
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 500.4285888671875,
      "epoch": 0.11435832274459974,
      "grad_norm": 8.883991241455078,
      "kl": 0.9453125,
      "learning_rate": 3.832668375104312e-06,
      "loss": 0.0095,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 180
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 519.0,
      "epoch": 0.11499364675984752,
      "grad_norm": 12.942805290222168,
      "kl": 1.375,
      "learning_rate": 3.8198695626733725e-06,
      "loss": 0.0141,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 181
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 667.2857666015625,
      "epoch": 0.1156289707750953,
      "grad_norm": 2.5961642265319824,
      "kl": 0.23828125,
      "learning_rate": 3.8070258606583156e-06,
      "loss": 0.0024,
      "reward": 0.3142857253551483,
      "reward_std": 0.10690449923276901,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 182
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 587.2857666015625,
      "epoch": 0.11626429479034307,
      "grad_norm": 3.12656831741333,
      "kl": 0.251953125,
      "learning_rate": 3.7941378079551544e-06,
      "loss": 0.0025,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 183
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 328.71429443359375,
      "epoch": 0.11689961880559085,
      "grad_norm": 2.924964427947998,
      "kl": 0.25,
      "learning_rate": 3.7812059453207677e-06,
      "loss": 0.0025,
      "reward": 0.6285714507102966,
      "reward_std": 0.5468525290489197,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 0.8571429252624512,
      "step": 184
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 486.2857360839844,
      "epoch": 0.11753494282083862,
      "grad_norm": 3.5567517280578613,
      "kl": 0.296875,
      "learning_rate": 3.768230815350213e-06,
      "loss": 0.003,
      "reward": 0.37142857909202576,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 185
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 554.1428833007812,
      "epoch": 0.1181702668360864,
      "grad_norm": 2.51031494140625,
      "kl": 0.30859375,
      "learning_rate": 3.7552129624539557e-06,
      "loss": 0.0031,
      "reward": 0.37142857909202576,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 186
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 594.5714721679688,
      "epoch": 0.11880559085133419,
      "grad_norm": 2.589846611022949,
      "kl": 0.26171875,
      "learning_rate": 3.7421529328350316e-06,
      "loss": 0.0026,
      "reward": 0.5284404158592224,
      "reward_std": 0.19612440466880798,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.12844036519527435,
      "rewards/format_reward": 1.0,
      "step": 187
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 595.857177734375,
      "epoch": 0.11944091486658195,
      "grad_norm": 2.4991862773895264,
      "kl": 0.27734375,
      "learning_rate": 3.7290512744661274e-06,
      "loss": 0.0028,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 188
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 552.0,
      "epoch": 0.12007623888182974,
      "grad_norm": 2.0362820625305176,
      "kl": 0.1845703125,
      "learning_rate": 3.715908537066589e-06,
      "loss": 0.0019,
      "reward": 0.485714316368103,
      "reward_std": 0.42983949184417725,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 189
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 698.5714721679688,
      "epoch": 0.1207115628970775,
      "grad_norm": 2.4349231719970703,
      "kl": 0.25,
      "learning_rate": 3.7027252720793538e-06,
      "loss": 0.0025,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 190
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 453.2857360839844,
      "epoch": 0.12134688691232529,
      "grad_norm": 3.212773323059082,
      "kl": 0.1796875,
      "learning_rate": 3.689502032647817e-06,
      "loss": 0.0018,
      "reward": 0.36014559864997864,
      "reward_std": 0.1179879903793335,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.017288444563746452,
      "rewards/format_reward": 1.0,
      "step": 191
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 671.1428833007812,
      "epoch": 0.12198221092757307,
      "grad_norm": 0.08436475694179535,
      "kl": 0.1767578125,
      "learning_rate": 3.6762393735926245e-06,
      "loss": 0.0021,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 192
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 651.2857666015625,
      "epoch": 0.12261753494282084,
      "grad_norm": 2.7841696739196777,
      "kl": 0.146484375,
      "learning_rate": 3.6629378513883852e-06,
      "loss": 0.0015,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 193
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 545.2857666015625,
      "epoch": 0.12325285895806862,
      "grad_norm": 3.3902151584625244,
      "kl": 0.205078125,
      "learning_rate": 3.6495980241403307e-06,
      "loss": 0.002,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 194
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 593.1428833007812,
      "epoch": 0.12388818297331639,
      "grad_norm": 2.5586631298065186,
      "kl": 0.197265625,
      "learning_rate": 3.636220451560896e-06,
      "loss": 0.002,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 195
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 589.7142944335938,
      "epoch": 0.12452350698856417,
      "grad_norm": 0.08236116170883179,
      "kl": 0.3203125,
      "learning_rate": 3.622805694946235e-06,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 196
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 780.7142944335938,
      "epoch": 0.12515883100381195,
      "grad_norm": 0.9724776148796082,
      "kl": 0.390625,
      "learning_rate": 3.609354317152667e-06,
      "loss": 0.0042,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 197
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 921.5714721679688,
      "epoch": 0.12579415501905972,
      "grad_norm": 2.485982894897461,
      "kl": 0.263671875,
      "learning_rate": 3.595866882573063e-06,
      "loss": 0.0026,
      "reward": 0.20000001788139343,
      "reward_std": 0.20000001788139343,
      "rewards/code_format_reward": 0.4285714626312256,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 198
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 850.2857666015625,
      "epoch": 0.12642947903430748,
      "grad_norm": 2.52078914642334,
      "kl": 0.451171875,
      "learning_rate": 3.5823439571131675e-06,
      "loss": 0.0045,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 199
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 924.2857666015625,
      "epoch": 0.12706480304955528,
      "grad_norm": 2.445126533508301,
      "kl": 0.20703125,
      "learning_rate": 3.5687861081678477e-06,
      "loss": 0.0021,
      "reward": 0.22857144474983215,
      "reward_std": 0.21380899846553802,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.5714285969734192,
      "step": 200
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 764.1428833007812,
      "epoch": 0.12770012706480305,
      "grad_norm": 2.1626222133636475,
      "kl": 0.3359375,
      "learning_rate": 3.555193904597291e-06,
      "loss": 0.0034,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 201
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 798.2857666015625,
      "epoch": 0.12833545108005082,
      "grad_norm": 2.010795831680298,
      "kl": 0.275390625,
      "learning_rate": 3.541567916703138e-06,
      "loss": 0.0027,
      "reward": 0.485714316368103,
      "reward_std": 0.42983949184417725,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 202
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 675.5714721679688,
      "epoch": 0.1289707750952986,
      "grad_norm": 2.5785133838653564,
      "kl": 0.2431640625,
      "learning_rate": 3.5279087162045517e-06,
      "loss": 0.0024,
      "reward": 0.6285714507102966,
      "reward_std": 0.5468525290489197,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 0.8571429252624512,
      "step": 203
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 696.4285888671875,
      "epoch": 0.12960609911054638,
      "grad_norm": 0.14385801553726196,
      "kl": 0.166015625,
      "learning_rate": 3.5142168762142265e-06,
      "loss": 0.002,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 204
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 673.5714721679688,
      "epoch": 0.13024142312579415,
      "grad_norm": 29.659584045410156,
      "kl": 2.109375,
      "learning_rate": 3.500492971214347e-06,
      "loss": 0.0212,
      "reward": 0.40446433424949646,
      "reward_std": 0.00819578766822815,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.004464285913854837,
      "rewards/format_reward": 1.0,
      "step": 205
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 724.857177734375,
      "epoch": 0.13087674714104194,
      "grad_norm": 3.008819580078125,
      "kl": 0.29296875,
      "learning_rate": 3.48673757703248e-06,
      "loss": 0.0029,
      "reward": 0.4012531638145447,
      "reward_std": 0.003315483685582876,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0012531329412013292,
      "rewards/format_reward": 1.0,
      "step": 206
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 670.0,
      "epoch": 0.1315120711562897,
      "grad_norm": 2.4119279384613037,
      "kl": 0.1904296875,
      "learning_rate": 3.472951270817418e-06,
      "loss": 0.0019,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 207
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 723.5714721679688,
      "epoch": 0.13214739517153748,
      "grad_norm": 0.09062952548265457,
      "kl": 0.26171875,
      "learning_rate": 3.4591346310149578e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 208
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 885.5714721679688,
      "epoch": 0.13278271918678525,
      "grad_norm": 2.048800468444824,
      "kl": 0.2109375,
      "learning_rate": 3.445288237343632e-06,
      "loss": 0.0021,
      "reward": 0.2857142984867096,
      "reward_std": 0.1864454597234726,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.02857143059372902,
      "rewards/format_reward": 0.7142857313156128,
      "step": 209
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 768.0000610351562,
      "epoch": 0.13341804320203304,
      "grad_norm": 0.07965610176324844,
      "kl": 0.279296875,
      "learning_rate": 3.4314126707703895e-06,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 210
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 668.7142944335938,
      "epoch": 0.1340533672172808,
      "grad_norm": 2.6163687705993652,
      "kl": 0.259765625,
      "learning_rate": 3.4175085134862128e-06,
      "loss": 0.0026,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 211
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 846.5714721679688,
      "epoch": 0.13468869123252858,
      "grad_norm": 2.003486394882202,
      "kl": 0.1591796875,
      "learning_rate": 3.4035763488816953e-06,
      "loss": 0.0016,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 212
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 518.5714721679688,
      "epoch": 0.13532401524777637,
      "grad_norm": 2.321255683898926,
      "kl": 0.1865234375,
      "learning_rate": 3.3896167615225594e-06,
      "loss": 0.0019,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 213
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 736.4285888671875,
      "epoch": 0.13595933926302414,
      "grad_norm": 0.06602618098258972,
      "kl": 0.25,
      "learning_rate": 3.375630337125133e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 214
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 668.2857666015625,
      "epoch": 0.1365946632782719,
      "grad_norm": 2.6393556594848633,
      "kl": 0.18359375,
      "learning_rate": 3.361617662531772e-06,
      "loss": 0.0018,
      "reward": 0.5445378422737122,
      "reward_std": 0.2468453347682953,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.14453783631324768,
      "rewards/format_reward": 1.0,
      "step": 215
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 696.5714721679688,
      "epoch": 0.1372299872935197,
      "grad_norm": 0.06534316390752792,
      "kl": 0.1962890625,
      "learning_rate": 3.347579325686237e-06,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 216
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 728.4285888671875,
      "epoch": 0.13786531130876747,
      "grad_norm": 2.1525208950042725,
      "kl": 0.162109375,
      "learning_rate": 3.333515915609027e-06,
      "loss": 0.0016,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 217
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 695.5714721679688,
      "epoch": 0.13850063532401524,
      "grad_norm": 2.821530342102051,
      "kl": 0.271484375,
      "learning_rate": 3.3194280223726616e-06,
      "loss": 0.0027,
      "reward": 0.41904765367507935,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0476190522313118,
      "rewards/format_reward": 1.0,
      "step": 218
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 558.4285888671875,
      "epoch": 0.13913595933926304,
      "grad_norm": 0.07845824211835861,
      "kl": 0.322265625,
      "learning_rate": 3.305316237076927e-06,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 219
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 762.2857666015625,
      "epoch": 0.1397712833545108,
      "grad_norm": 2.1945505142211914,
      "kl": 0.15625,
      "learning_rate": 3.291181151824071e-06,
      "loss": 0.0016,
      "reward": 0.3142857253551483,
      "reward_std": 0.15735916793346405,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 220
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 534.0,
      "epoch": 0.14040660736975857,
      "grad_norm": 2.6735599040985107,
      "kl": 0.19921875,
      "learning_rate": 3.27702335969396e-06,
      "loss": 0.002,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 221
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 673.2857666015625,
      "epoch": 0.14104193138500634,
      "grad_norm": 0.16328755021095276,
      "kl": 0.3203125,
      "learning_rate": 3.2628434547191985e-06,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 222
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 690.7142944335938,
      "epoch": 0.14167725540025414,
      "grad_norm": 0.05350850895047188,
      "kl": 0.28515625,
      "learning_rate": 3.2486420318601973e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 223
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 705.7142944335938,
      "epoch": 0.1423125794155019,
      "grad_norm": 0.05627477914094925,
      "kl": 0.255859375,
      "learning_rate": 3.2344196869802187e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 224
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 522.857177734375,
      "epoch": 0.14294790343074967,
      "grad_norm": 3.546515464782715,
      "kl": 0.1943359375,
      "learning_rate": 3.2201770168203694e-06,
      "loss": 0.0019,
      "reward": 0.37142857909202576,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 225
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 656.857177734375,
      "epoch": 0.14358322744599747,
      "grad_norm": 0.30215829610824585,
      "kl": 0.283203125,
      "learning_rate": 3.205914618974563e-06,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 226
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 680.1428833007812,
      "epoch": 0.14421855146124524,
      "grad_norm": 2.333859443664551,
      "kl": 0.1982421875,
      "learning_rate": 3.1916330918644496e-06,
      "loss": 0.002,
      "reward": 0.290529727935791,
      "reward_std": 0.16188988089561462,
      "rewards/code_format_reward": 0.5714285969734192,
      "rewards/code_reward": 0.004815409425646067,
      "rewards/format_reward": 0.8571429252624512,
      "step": 227
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 702.7142944335938,
      "epoch": 0.144853875476493,
      "grad_norm": 0.06898491084575653,
      "kl": 0.26171875,
      "learning_rate": 3.177333034714303e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 228
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 699.7142944335938,
      "epoch": 0.1454891994917408,
      "grad_norm": 3.096179246902466,
      "kl": 0.26171875,
      "learning_rate": 3.1630150475258813e-06,
      "loss": 0.0026,
      "reward": 0.3428571820259094,
      "reward_std": 0.09759000688791275,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 229
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 704.4285888671875,
      "epoch": 0.14612452350698857,
      "grad_norm": 0.06554409861564636,
      "kl": 0.302734375,
      "learning_rate": 3.148679731053252e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 230
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 575.5714721679688,
      "epoch": 0.14675984752223634,
      "grad_norm": 2.8049392700195312,
      "kl": 0.2158203125,
      "learning_rate": 3.1343276867775805e-06,
      "loss": 0.0022,
      "reward": 0.4242587983608246,
      "reward_std": 0.04178621619939804,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.02425876073539257,
      "rewards/format_reward": 1.0,
      "step": 231
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 611.857177734375,
      "epoch": 0.1473951715374841,
      "grad_norm": 3.6352577209472656,
      "kl": 0.296875,
      "learning_rate": 3.1199595168819043e-06,
      "loss": 0.003,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 232
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 638.0,
      "epoch": 0.1480304955527319,
      "grad_norm": 2.4661004543304443,
      "kl": 0.298828125,
      "learning_rate": 3.105575824225852e-06,
      "loss": 0.003,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 233
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.857177734375,
      "epoch": 0.14866581956797967,
      "grad_norm": 2.9908604621887207,
      "kl": 0.20703125,
      "learning_rate": 3.091177212320363e-06,
      "loss": 0.0021,
      "reward": 0.37142857909202576,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 234
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 652.1428833007812,
      "epoch": 0.14930114358322744,
      "grad_norm": 0.04900944232940674,
      "kl": 0.1875,
      "learning_rate": 3.0767642853023538e-06,
      "loss": 0.0022,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 235
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 485.5714416503906,
      "epoch": 0.14993646759847523,
      "grad_norm": 2.6988017559051514,
      "kl": 0.216796875,
      "learning_rate": 3.062337647909376e-06,
      "loss": 0.0022,
      "reward": 0.37142857909202576,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 236
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 672.857177734375,
      "epoch": 0.150571791613723,
      "grad_norm": 2.6142776012420654,
      "kl": 0.22265625,
      "learning_rate": 3.04789790545424e-06,
      "loss": 0.0022,
      "reward": 0.4169172942638397,
      "reward_std": 0.04475894197821617,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.016917293891310692,
      "rewards/format_reward": 1.0,
      "step": 237
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 709.7142944335938,
      "epoch": 0.15120711562897077,
      "grad_norm": 0.06715590506792068,
      "kl": 0.1865234375,
      "learning_rate": 3.033445663799621e-06,
      "loss": 0.0022,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 238
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 642.2857666015625,
      "epoch": 0.15184243964421856,
      "grad_norm": 2.8286352157592773,
      "kl": 0.2197265625,
      "learning_rate": 3.018981529332633e-06,
      "loss": 0.0022,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645264148712,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 239
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 567.2857666015625,
      "epoch": 0.15247776365946633,
      "grad_norm": 0.07005994766950607,
      "kl": 0.21875,
      "learning_rate": 3.00450610893939e-06,
      "loss": 0.0025,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 240
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 678.5714721679688,
      "epoch": 0.1531130876747141,
      "grad_norm": 2.2401628494262695,
      "kl": 0.162109375,
      "learning_rate": 2.9900200099795396e-06,
      "loss": 0.0016,
      "reward": 0.40160515904426575,
      "reward_std": 0.004246791359037161,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.00160513655282557,
      "rewards/format_reward": 1.0,
      "step": 241
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 640.4285888671875,
      "epoch": 0.15374841168996187,
      "grad_norm": 2.341778039932251,
      "kl": 0.30078125,
      "learning_rate": 2.9755238402607826e-06,
      "loss": 0.003,
      "reward": 0.485714316368103,
      "reward_std": 0.22677868604660034,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 242
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 929.4285888671875,
      "epoch": 0.15438373570520966,
      "grad_norm": 2.2782864570617676,
      "kl": 0.193359375,
      "learning_rate": 2.961018208013367e-06,
      "loss": 0.0019,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 243
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 655.5714721679688,
      "epoch": 0.15501905972045743,
      "grad_norm": 0.0515868104994297,
      "kl": 0.265625,
      "learning_rate": 2.9465037218645694e-06,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 244
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 596.7142944335938,
      "epoch": 0.1556543837357052,
      "grad_norm": 0.06041451543569565,
      "kl": 0.271484375,
      "learning_rate": 2.9319809908131604e-06,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 245
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 623.5714721679688,
      "epoch": 0.156289707750953,
      "grad_norm": 0.11632593721151352,
      "kl": 0.33203125,
      "learning_rate": 2.917450624203847e-06,
      "loss": 0.0036,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 246
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 708.7142944335938,
      "epoch": 0.15692503176620076,
      "grad_norm": 2.282824993133545,
      "kl": 0.17578125,
      "learning_rate": 2.9029132317017118e-06,
      "loss": 0.0018,
      "reward": 0.6714285612106323,
      "reward_std": 0.27516230940818787,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.30000001192092896,
      "rewards/format_reward": 1.0,
      "step": 247
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.2857360839844,
      "epoch": 0.15756035578144853,
      "grad_norm": 0.05820649862289429,
      "kl": 0.263671875,
      "learning_rate": 2.888369423266629e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 248
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 559.1428833007812,
      "epoch": 0.15819567979669633,
      "grad_norm": 2.360961437225342,
      "kl": 0.2431640625,
      "learning_rate": 2.8738198091276712e-06,
      "loss": 0.0024,
      "reward": 0.771428644657135,
      "reward_std": 0.48205915093421936,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 0.8571429252624512,
      "step": 249
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 673.857177734375,
      "epoch": 0.1588310038119441,
      "grad_norm": 2.426175832748413,
      "kl": 0.2431640625,
      "learning_rate": 2.859264999757509e-06,
      "loss": 0.0024,
      "reward": 0.485714316368103,
      "reward_std": 0.42983946204185486,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 250
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 555.7142944335938,
      "epoch": 0.15946632782719186,
      "grad_norm": 1.7476675510406494,
      "kl": 0.2060546875,
      "learning_rate": 2.8447056058467928e-06,
      "loss": 0.0021,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 251
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 635.1428833007812,
      "epoch": 0.16010165184243966,
      "grad_norm": 0.10966142266988754,
      "kl": 0.296875,
      "learning_rate": 2.830142238278531e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 252
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 569.4285888671875,
      "epoch": 0.16073697585768743,
      "grad_norm": 0.07483859360218048,
      "kl": 0.306640625,
      "learning_rate": 2.81557550810246e-06,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 253
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 696.1428833007812,
      "epoch": 0.1613722998729352,
      "grad_norm": 557283.4375,
      "kl": 19456.0,
      "learning_rate": 2.8010060265094026e-06,
      "loss": 194.2972,
      "reward": 0.485714316368103,
      "reward_std": 0.42983946204185486,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 254
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 572.1428833007812,
      "epoch": 0.16200762388818296,
      "grad_norm": 2.958310842514038,
      "kl": 0.203125,
      "learning_rate": 2.786434404805629e-06,
      "loss": 0.002,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 255
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 642.5714721679688,
      "epoch": 0.16264294790343076,
      "grad_norm": 2.9970836639404297,
      "kl": 0.359375,
      "learning_rate": 2.771861254387199e-06,
      "loss": 0.0036,
      "reward": 0.485714316368103,
      "reward_std": 0.42983949184417725,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 0.8571429252624512,
      "step": 256
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 626.5714721679688,
      "epoch": 0.16327827191867852,
      "grad_norm": 0.052250247448682785,
      "kl": 0.203125,
      "learning_rate": 2.7572871867143204e-06,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 257
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 587.0,
      "epoch": 0.1639135959339263,
      "grad_norm": 0.049148622900247574,
      "kl": 0.1845703125,
      "learning_rate": 2.742712813285681e-06,
      "loss": 0.0021,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 258
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 413.71429443359375,
      "epoch": 0.1645489199491741,
      "grad_norm": 3.1558637619018555,
      "kl": 0.296875,
      "learning_rate": 2.7281387456128017e-06,
      "loss": 0.003,
      "reward": 0.6661654710769653,
      "reward_std": 0.2892994284629822,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.3233082890510559,
      "rewards/format_reward": 0.8571429252624512,
      "step": 259
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 599.1428833007812,
      "epoch": 0.16518424396442186,
      "grad_norm": 2.672922372817993,
      "kl": 0.333984375,
      "learning_rate": 2.7135655951943716e-06,
      "loss": 0.0033,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 260
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 638.2857666015625,
      "epoch": 0.16581956797966962,
      "grad_norm": 0.062102027237415314,
      "kl": 0.2890625,
      "learning_rate": 2.698993973490598e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 261
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 615.4285888671875,
      "epoch": 0.16645489199491742,
      "grad_norm": 2.032621145248413,
      "kl": 0.349609375,
      "learning_rate": 2.6844244918975416e-06,
      "loss": 0.0035,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 262
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 680.4285888671875,
      "epoch": 0.1670902160101652,
      "grad_norm": 0.07202436029911041,
      "kl": 0.25390625,
      "learning_rate": 2.66985776172147e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 263
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 607.0,
      "epoch": 0.16772554002541296,
      "grad_norm": 3.1766250133514404,
      "kl": 0.3046875,
      "learning_rate": 2.6552943941532088e-06,
      "loss": 0.003,
      "reward": 0.41587308049201965,
      "reward_std": 0.0419960655272007,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.01587301678955555,
      "rewards/format_reward": 1.0,
      "step": 264
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 661.5714721679688,
      "epoch": 0.16836086404066072,
      "grad_norm": 3.2823150157928467,
      "kl": 0.2392578125,
      "learning_rate": 2.6407350002424927e-06,
      "loss": 0.0024,
      "reward": 0.5868132710456848,
      "reward_std": 0.23857378959655762,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.18681320548057556,
      "rewards/format_reward": 1.0,
      "step": 265
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 598.857177734375,
      "epoch": 0.16899618805590852,
      "grad_norm": 0.08283974230289459,
      "kl": 0.2021484375,
      "learning_rate": 2.626180190872329e-06,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 266
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.2857360839844,
      "epoch": 0.1696315120711563,
      "grad_norm": 3.3979873657226562,
      "kl": 0.234375,
      "learning_rate": 2.611630576733372e-06,
      "loss": 0.0023,
      "reward": 0.41260507702827454,
      "reward_std": 0.01228986494243145,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.01260504312813282,
      "rewards/format_reward": 1.0,
      "step": 267
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 584.1428833007812,
      "epoch": 0.17026683608640406,
      "grad_norm": 2.4819862842559814,
      "kl": 0.28125,
      "learning_rate": 2.5970867682982885e-06,
      "loss": 0.0028,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 268
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 458.14288330078125,
      "epoch": 0.17090216010165185,
      "grad_norm": 0.08856673538684845,
      "kl": 0.349609375,
      "learning_rate": 2.582549375796154e-06,
      "loss": 0.0038,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 269
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 468.71429443359375,
      "epoch": 0.17153748411689962,
      "grad_norm": 2.835487127304077,
      "kl": 0.32421875,
      "learning_rate": 2.568019009186841e-06,
      "loss": 0.0032,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 270
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 637.4285888671875,
      "epoch": 0.1721728081321474,
      "grad_norm": 0.054750654846429825,
      "kl": 0.162109375,
      "learning_rate": 2.5534962781354317e-06,
      "loss": 0.0019,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 271
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 757.2857666015625,
      "epoch": 0.17280813214739518,
      "grad_norm": 2.1099636554718018,
      "kl": 0.21484375,
      "learning_rate": 2.538981791986634e-06,
      "loss": 0.0021,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 272
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.2857360839844,
      "epoch": 0.17344345616264295,
      "grad_norm": 2.960906982421875,
      "kl": 0.251953125,
      "learning_rate": 2.524476159739218e-06,
      "loss": 0.0025,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645264148712,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 273
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 592.5714721679688,
      "epoch": 0.17407878017789072,
      "grad_norm": 2.5509631633758545,
      "kl": 0.26953125,
      "learning_rate": 2.5099799900204607e-06,
      "loss": 0.0027,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 274
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 436.8571472167969,
      "epoch": 0.17471410419313851,
      "grad_norm": 2.9435410499572754,
      "kl": 0.400390625,
      "learning_rate": 2.4954938910606108e-06,
      "loss": 0.004,
      "reward": 0.563265323638916,
      "reward_std": 0.37278667092323303,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.16326531767845154,
      "rewards/format_reward": 1.0,
      "step": 275
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 409.5714416503906,
      "epoch": 0.17534942820838628,
      "grad_norm": 4.025363922119141,
      "kl": 0.275390625,
      "learning_rate": 2.481018470667368e-06,
      "loss": 0.0028,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 276
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 467.14288330078125,
      "epoch": 0.17598475222363405,
      "grad_norm": 0.09415756165981293,
      "kl": 0.30078125,
      "learning_rate": 2.4665543362003802e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 277
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.0000305175781,
      "epoch": 0.17662007623888182,
      "grad_norm": 3.590571165084839,
      "kl": 0.2216796875,
      "learning_rate": 2.4521020945457615e-06,
      "loss": 0.0022,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 278
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.2857360839844,
      "epoch": 0.1772554002541296,
      "grad_norm": 2.6514501571655273,
      "kl": 0.1884765625,
      "learning_rate": 2.4376623520906255e-06,
      "loss": 0.0019,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 279
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.4285888671875,
      "epoch": 0.17789072426937738,
      "grad_norm": 3.0715246200561523,
      "kl": 0.396484375,
      "learning_rate": 2.4232357146976478e-06,
      "loss": 0.004,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 280
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 552.1428833007812,
      "epoch": 0.17852604828462515,
      "grad_norm": 0.04878819361329079,
      "kl": 0.25390625,
      "learning_rate": 2.408822787679637e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 281
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 470.0000305175781,
      "epoch": 0.17916137229987295,
      "grad_norm": 2.8764052391052246,
      "kl": 0.337890625,
      "learning_rate": 2.3944241757741475e-06,
      "loss": 0.0034,
      "reward": 0.7617021799087524,
      "reward_std": 0.39519527554512024,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.3617021143436432,
      "rewards/format_reward": 1.0,
      "step": 282
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 552.0,
      "epoch": 0.1797966963151207,
      "grad_norm": 3.4082202911376953,
      "kl": 0.279296875,
      "learning_rate": 2.380040483118097e-06,
      "loss": 0.0028,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 283
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.0000305175781,
      "epoch": 0.18043202033036848,
      "grad_norm": 3.0144095420837402,
      "kl": 0.236328125,
      "learning_rate": 2.365672313222419e-06,
      "loss": 0.0024,
      "reward": 0.800332248210907,
      "reward_std": 0.2776820659637451,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4003322720527649,
      "rewards/format_reward": 1.0,
      "step": 284
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 514.5714721679688,
      "epoch": 0.18106734434561628,
      "grad_norm": 0.1193128377199173,
      "kl": 0.35546875,
      "learning_rate": 2.351320268946749e-06,
      "loss": 0.0038,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 285
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 638.7142944335938,
      "epoch": 0.18170266836086404,
      "grad_norm": 0.050928860902786255,
      "kl": 0.25390625,
      "learning_rate": 2.336984952474119e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 286
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 496.857177734375,
      "epoch": 0.1823379923761118,
      "grad_norm": 0.12127784639596939,
      "kl": 0.310546875,
      "learning_rate": 2.322666965285697e-06,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 287
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 515.857177734375,
      "epoch": 0.18297331639135958,
      "grad_norm": 0.04745308309793472,
      "kl": 0.302734375,
      "learning_rate": 2.3083669081355507e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 288
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 619.0,
      "epoch": 0.18360864040660738,
      "grad_norm": 0.04747169092297554,
      "kl": 0.2392578125,
      "learning_rate": 2.2940853810254377e-06,
      "loss": 0.0027,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 289
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 427.2857360839844,
      "epoch": 0.18424396442185514,
      "grad_norm": 0.04503343254327774,
      "kl": 0.2109375,
      "learning_rate": 2.2798229831796313e-06,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 290
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 350.8571472167969,
      "epoch": 0.1848792884371029,
      "grad_norm": 0.15933562815189362,
      "kl": 0.29296875,
      "learning_rate": 2.2655803130197816e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 291
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 492.4285888671875,
      "epoch": 0.1855146124523507,
      "grad_norm": 0.061849016696214676,
      "kl": 0.34375,
      "learning_rate": 2.2513579681398034e-06,
      "loss": 0.0037,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 292
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 447.5714416503906,
      "epoch": 0.18614993646759848,
      "grad_norm": 3.1712300777435303,
      "kl": 0.1982421875,
      "learning_rate": 2.237156545280803e-06,
      "loss": 0.002,
      "reward": 1.1142858266830444,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.7142857313156128,
      "rewards/format_reward": 1.0,
      "step": 293
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 530.7142944335938,
      "epoch": 0.18678526048284624,
      "grad_norm": 2.926269769668579,
      "kl": 0.27734375,
      "learning_rate": 2.2229766403060403e-06,
      "loss": 0.0028,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 294
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 370.14288330078125,
      "epoch": 0.18742058449809404,
      "grad_norm": 3.639002799987793,
      "kl": 0.412109375,
      "learning_rate": 2.2088188481759305e-06,
      "loss": 0.0041,
      "reward": 0.44044750928878784,
      "reward_std": 0.06550441682338715,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.040447503328323364,
      "rewards/format_reward": 1.0,
      "step": 295
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.8571472167969,
      "epoch": 0.1880559085133418,
      "grad_norm": 3.473376512527466,
      "kl": 0.337890625,
      "learning_rate": 2.194683762923073e-06,
      "loss": 0.0034,
      "reward": 0.8047619462013245,
      "reward_std": 0.28637492656707764,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4047619104385376,
      "rewards/format_reward": 1.0,
      "step": 296
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 537.1428833007812,
      "epoch": 0.18869123252858958,
      "grad_norm": 0.05299937725067139,
      "kl": 0.2373046875,
      "learning_rate": 2.1805719776273387e-06,
      "loss": 0.0027,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 297
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 499.0000305175781,
      "epoch": 0.18932655654383734,
      "grad_norm": 0.04631977900862694,
      "kl": 0.2265625,
      "learning_rate": 2.166484084390974e-06,
      "loss": 0.0026,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 298
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 490.14288330078125,
      "epoch": 0.18996188055908514,
      "grad_norm": 0.08931510150432587,
      "kl": 0.244140625,
      "learning_rate": 2.1524206743137636e-06,
      "loss": 0.0027,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 299
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 461.2857360839844,
      "epoch": 0.1905972045743329,
      "grad_norm": 3.1879539489746094,
      "kl": 0.236328125,
      "learning_rate": 2.1383823374682287e-06,
      "loss": 0.0024,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 300
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 493.14288330078125,
      "epoch": 0.19123252858958067,
      "grad_norm": 2.5813148021698,
      "kl": 0.2294921875,
      "learning_rate": 2.124369662874868e-06,
      "loss": 0.0023,
      "reward": 0.44916945695877075,
      "reward_std": 0.2263808697462082,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.10631229728460312,
      "rewards/format_reward": 0.8571429252624512,
      "step": 301
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 467.857177734375,
      "epoch": 0.19186785260482847,
      "grad_norm": 2.3633549213409424,
      "kl": 0.251953125,
      "learning_rate": 2.110383238477441e-06,
      "loss": 0.0025,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 302
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 467.5714416503906,
      "epoch": 0.19250317662007624,
      "grad_norm": 3.2412614822387695,
      "kl": 0.2021484375,
      "learning_rate": 2.096423651118305e-06,
      "loss": 0.002,
      "reward": 0.41476020216941833,
      "reward_std": 0.03590288758277893,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.014760148711502552,
      "rewards/format_reward": 1.0,
      "step": 303
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 629.1428833007812,
      "epoch": 0.193138500635324,
      "grad_norm": 1.8208192586898804,
      "kl": 0.1845703125,
      "learning_rate": 2.082491486513788e-06,
      "loss": 0.0018,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 304
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 573.1428833007812,
      "epoch": 0.1937738246505718,
      "grad_norm": 0.06740770488977432,
      "kl": 0.326171875,
      "learning_rate": 2.0685873292296116e-06,
      "loss": 0.0036,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 305
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 469.0000305175781,
      "epoch": 0.19440914866581957,
      "grad_norm": 0.050011664628982544,
      "kl": 0.322265625,
      "learning_rate": 2.054711762656369e-06,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 306
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 501.2857360839844,
      "epoch": 0.19504447268106734,
      "grad_norm": 2.7797482013702393,
      "kl": 0.1865234375,
      "learning_rate": 2.040865368985044e-06,
      "loss": 0.0019,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 307
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 483.4285888671875,
      "epoch": 0.19567979669631513,
      "grad_norm": 0.056970253586769104,
      "kl": 0.205078125,
      "learning_rate": 2.027048729182583e-06,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 308
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 708.4285888671875,
      "epoch": 0.1963151207115629,
      "grad_norm": 1.9797154664993286,
      "kl": 0.2421875,
      "learning_rate": 2.0132624229675205e-06,
      "loss": 0.0024,
      "reward": 0.45210087299346924,
      "reward_std": 0.11403417587280273,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.10924370586872101,
      "rewards/format_reward": 0.8571429252624512,
      "step": 309
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.14288330078125,
      "epoch": 0.19695044472681067,
      "grad_norm": 0.0952640175819397,
      "kl": 0.40625,
      "learning_rate": 1.9995070287856546e-06,
      "loss": 0.0044,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 310
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 533.7142944335938,
      "epoch": 0.19758576874205844,
      "grad_norm": 3.1438472270965576,
      "kl": 0.30859375,
      "learning_rate": 1.985783123785774e-06,
      "loss": 0.0031,
      "reward": 0.49523812532424927,
      "reward_std": 0.11878278106451035,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0952381044626236,
      "rewards/format_reward": 1.0,
      "step": 311
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 536.5714721679688,
      "epoch": 0.19822109275730623,
      "grad_norm": 2.1929433345794678,
      "kl": 0.314453125,
      "learning_rate": 1.9720912837954486e-06,
      "loss": 0.0031,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 312
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 502.71429443359375,
      "epoch": 0.198856416772554,
      "grad_norm": 0.05079368129372597,
      "kl": 0.306640625,
      "learning_rate": 1.958432083296862e-06,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 313
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 454.14288330078125,
      "epoch": 0.19949174078780177,
      "grad_norm": 0.06545651704072952,
      "kl": 0.2099609375,
      "learning_rate": 1.9448060954027093e-06,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 314
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 523.1428833007812,
      "epoch": 0.20012706480304956,
      "grad_norm": 2.852233409881592,
      "kl": 0.27734375,
      "learning_rate": 1.931213891832153e-06,
      "loss": 0.0028,
      "reward": 0.40317460894584656,
      "reward_std": 0.005421662237495184,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0031746034510433674,
      "rewards/format_reward": 1.0,
      "step": 315
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 518.7142944335938,
      "epoch": 0.20076238881829733,
      "grad_norm": 0.056687433272600174,
      "kl": 0.2158203125,
      "learning_rate": 1.9176560428868336e-06,
      "loss": 0.0025,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 316
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 651.857177734375,
      "epoch": 0.2013977128335451,
      "grad_norm": 2.1868667602539062,
      "kl": 0.1923828125,
      "learning_rate": 1.9041331174269373e-06,
      "loss": 0.0019,
      "reward": 0.3556329905986786,
      "reward_std": 0.16032202541828156,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.012775842100381851,
      "rewards/format_reward": 0.8571429252624512,
      "step": 317
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 627.7142944335938,
      "epoch": 0.2020330368487929,
      "grad_norm": 0.055125512182712555,
      "kl": 0.2578125,
      "learning_rate": 1.8906456828473341e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 318
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 585.4285888671875,
      "epoch": 0.20266836086404066,
      "grad_norm": 0.0708109587430954,
      "kl": 0.265625,
      "learning_rate": 1.8771943050537656e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 319
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 568.857177734375,
      "epoch": 0.20330368487928843,
      "grad_norm": 3.287763833999634,
      "kl": 0.1796875,
      "learning_rate": 1.8637795484391046e-06,
      "loss": 0.0018,
      "reward": 0.4266955256462097,
      "reward_std": 0.05106709897518158,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.026695528998970985,
      "rewards/format_reward": 1.0,
      "step": 320
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 356.71429443359375,
      "epoch": 0.2039390088945362,
      "grad_norm": 2.707819700241089,
      "kl": 0.271484375,
      "learning_rate": 1.8504019758596698e-06,
      "loss": 0.0027,
      "reward": 1.2571430206298828,
      "reward_std": 0.3779645264148712,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.8571429252624512,
      "rewards/format_reward": 1.0,
      "step": 321
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 470.4285888671875,
      "epoch": 0.204574332909784,
      "grad_norm": 2.942082405090332,
      "kl": 0.353515625,
      "learning_rate": 1.8370621486116163e-06,
      "loss": 0.0035,
      "reward": 0.41904765367507935,
      "reward_std": 0.05039527267217636,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.01904762163758278,
      "rewards/format_reward": 1.0,
      "step": 322
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 485.14288330078125,
      "epoch": 0.20520965692503176,
      "grad_norm": 0.06295392662286758,
      "kl": 0.330078125,
      "learning_rate": 1.823760626407377e-06,
      "loss": 0.0036,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 323
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 499.4285888671875,
      "epoch": 0.20584498094027953,
      "grad_norm": 2.458465099334717,
      "kl": 0.34375,
      "learning_rate": 1.8104979673521838e-06,
      "loss": 0.0034,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 324
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 526.2857666015625,
      "epoch": 0.20648030495552733,
      "grad_norm": 3.1990699768066406,
      "kl": 0.33203125,
      "learning_rate": 1.7972747279206482e-06,
      "loss": 0.0033,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 325
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 601.0,
      "epoch": 0.2071156289707751,
      "grad_norm": 0.05853183940052986,
      "kl": 0.2890625,
      "learning_rate": 1.7840914629334122e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 326
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 498.71429443359375,
      "epoch": 0.20775095298602286,
      "grad_norm": 0.07322244346141815,
      "kl": 0.3125,
      "learning_rate": 1.7709487255338731e-06,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 327
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 553.857177734375,
      "epoch": 0.20838627700127066,
      "grad_norm": 0.17839957773685455,
      "kl": 0.228515625,
      "learning_rate": 1.7578470671649684e-06,
      "loss": 0.0026,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 328
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 530.857177734375,
      "epoch": 0.20902160101651843,
      "grad_norm": 0.1131465956568718,
      "kl": 0.3359375,
      "learning_rate": 1.744787037546045e-06,
      "loss": 0.0037,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 329
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 564.1428833007812,
      "epoch": 0.2096569250317662,
      "grad_norm": 0.08917635679244995,
      "kl": 0.30078125,
      "learning_rate": 1.731769184649788e-06,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 330
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.4285888671875,
      "epoch": 0.210292249047014,
      "grad_norm": 0.15413963794708252,
      "kl": 0.36328125,
      "learning_rate": 1.7187940546792325e-06,
      "loss": 0.0039,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 331
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 535.0,
      "epoch": 0.21092757306226176,
      "grad_norm": 3.1670172214508057,
      "kl": 0.306640625,
      "learning_rate": 1.7058621920448465e-06,
      "loss": 0.0031,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 332
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 477.14288330078125,
      "epoch": 0.21156289707750953,
      "grad_norm": 0.07635504752397537,
      "kl": 0.333984375,
      "learning_rate": 1.6929741393416855e-06,
      "loss": 0.0036,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 333
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 405.71429443359375,
      "epoch": 0.2121982210927573,
      "grad_norm": 0.0835573673248291,
      "kl": 0.361328125,
      "learning_rate": 1.6801304373266286e-06,
      "loss": 0.0039,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 334
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 556.0,
      "epoch": 0.2128335451080051,
      "grad_norm": 0.17580975592136383,
      "kl": 0.310546875,
      "learning_rate": 1.667331624895689e-06,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 335
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 382.14288330078125,
      "epoch": 0.21346886912325286,
      "grad_norm": 2.9555137157440186,
      "kl": 0.330078125,
      "learning_rate": 1.6545782390614037e-06,
      "loss": 0.0033,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 336
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.14288330078125,
      "epoch": 0.21410419313850063,
      "grad_norm": 0.08389069885015488,
      "kl": 0.2275390625,
      "learning_rate": 1.6418708149302992e-06,
      "loss": 0.0026,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 337
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.2857360839844,
      "epoch": 0.21473951715374842,
      "grad_norm": 2.6030876636505127,
      "kl": 0.23046875,
      "learning_rate": 1.6292098856804423e-06,
      "loss": 0.0023,
      "reward": 1.2571430206298828,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.8571429252624512,
      "rewards/format_reward": 1.0,
      "step": 338
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 369.71429443359375,
      "epoch": 0.2153748411689962,
      "grad_norm": 0.09957047551870346,
      "kl": 0.255859375,
      "learning_rate": 1.6165959825390661e-06,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 339
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 374.5714416503906,
      "epoch": 0.21601016518424396,
      "grad_norm": 3.5869476795196533,
      "kl": 0.2216796875,
      "learning_rate": 1.604029634760284e-06,
      "loss": 0.0022,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 340
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 449.14288330078125,
      "epoch": 0.21664548919949175,
      "grad_norm": 0.04904744401574135,
      "kl": 0.208984375,
      "learning_rate": 1.59151136960288e-06,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 341
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 592.0,
      "epoch": 0.21728081321473952,
      "grad_norm": 0.07155793160200119,
      "kl": 0.29296875,
      "learning_rate": 1.5790417123081903e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 342
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 498.14288330078125,
      "epoch": 0.2179161372299873,
      "grad_norm": 0.07623915374279022,
      "kl": 0.34765625,
      "learning_rate": 1.5666211860780583e-06,
      "loss": 0.0038,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 343
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.2857360839844,
      "epoch": 0.21855146124523506,
      "grad_norm": 3.3097286224365234,
      "kl": 0.330078125,
      "learning_rate": 1.5542503120528918e-06,
      "loss": 0.0033,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 344
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 504.4285888671875,
      "epoch": 0.21918678526048285,
      "grad_norm": 3.0239760875701904,
      "kl": 0.3125,
      "learning_rate": 1.5419296092897866e-06,
      "loss": 0.0031,
      "reward": 0.5071429014205933,
      "reward_std": 0.14202801883220673,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.107142873108387,
      "rewards/format_reward": 1.0,
      "step": 345
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 656.857177734375,
      "epoch": 0.21982210927573062,
      "grad_norm": 0.04605603963136673,
      "kl": 0.279296875,
      "learning_rate": 1.529659594740755e-06,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 346
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 411.2857360839844,
      "epoch": 0.2204574332909784,
      "grad_norm": 0.05368569865822792,
      "kl": 0.373046875,
      "learning_rate": 1.5174407832310338e-06,
      "loss": 0.004,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 347
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 626.0,
      "epoch": 0.22109275730622618,
      "grad_norm": 2.683948040008545,
      "kl": 0.26171875,
      "learning_rate": 1.5052736874374815e-06,
      "loss": 0.0026,
      "reward": 0.4285714626312256,
      "reward_std": 0.04879499599337578,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.02857143059372902,
      "rewards/format_reward": 1.0,
      "step": 348
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 500.857177734375,
      "epoch": 0.22172808132147395,
      "grad_norm": 2.875629186630249,
      "kl": 0.203125,
      "learning_rate": 1.4931588178670695e-06,
      "loss": 0.002,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 349
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 259.5714416503906,
      "epoch": 0.22236340533672172,
      "grad_norm": 0.2272838056087494,
      "kl": 0.353515625,
      "learning_rate": 1.4810966828354605e-06,
      "loss": 0.0047,
      "reward": 1.4000002145767212,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 1.0,
      "rewards/format_reward": 1.0,
      "step": 350
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 479.71429443359375,
      "epoch": 0.22299872935196952,
      "grad_norm": 3.2993814945220947,
      "kl": 0.21484375,
      "learning_rate": 1.469087788445684e-06,
      "loss": 0.0021,
      "reward": 0.42500001192092896,
      "reward_std": 0.04564352706074715,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.02500000223517418,
      "rewards/format_reward": 1.0,
      "step": 351
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 498.5714416503906,
      "epoch": 0.22363405336721728,
      "grad_norm": 0.2275489717721939,
      "kl": 0.333984375,
      "learning_rate": 1.4571326385668965e-06,
      "loss": 0.0036,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 352
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 470.857177734375,
      "epoch": 0.22426937738246505,
      "grad_norm": 0.05804005637764931,
      "kl": 0.341796875,
      "learning_rate": 1.4452317348132434e-06,
      "loss": 0.0037,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 353
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 255.1428680419922,
      "epoch": 0.22490470139771285,
      "grad_norm": 0.30045151710510254,
      "kl": 0.3125,
      "learning_rate": 1.4333855765228104e-06,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 354
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 401.8571472167969,
      "epoch": 0.22554002541296062,
      "grad_norm": 0.1624024361371994,
      "kl": 0.36328125,
      "learning_rate": 1.421594660736675e-06,
      "loss": 0.0039,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 355
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 458.5714416503906,
      "epoch": 0.22617534942820838,
      "grad_norm": 0.06793898344039917,
      "kl": 0.2294921875,
      "learning_rate": 1.4098594821780476e-06,
      "loss": 0.0026,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 356
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 354.2857360839844,
      "epoch": 0.22681067344345615,
      "grad_norm": 3.499160051345825,
      "kl": 0.2412109375,
      "learning_rate": 1.3981805332315174e-06,
      "loss": 0.0024,
      "reward": 1.057142972946167,
      "reward_std": 0.47207754850387573,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.7142857313156128,
      "rewards/format_reward": 0.8571429252624512,
      "step": 357
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.71429443359375,
      "epoch": 0.22744599745870395,
      "grad_norm": 3.1533403396606445,
      "kl": 0.30859375,
      "learning_rate": 1.3865583039223929e-06,
      "loss": 0.0031,
      "reward": 0.7381389141082764,
      "reward_std": 0.2022811770439148,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.3381389379501343,
      "rewards/format_reward": 1.0,
      "step": 358
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 287.8571472167969,
      "epoch": 0.22808132147395171,
      "grad_norm": 0.08542405813932419,
      "kl": 0.3203125,
      "learning_rate": 1.374993281896137e-06,
      "loss": 0.0044,
      "reward": 1.4000002145767212,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 1.0,
      "rewards/format_reward": 1.0,
      "step": 359
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 557.5714721679688,
      "epoch": 0.22871664548919948,
      "grad_norm": 1.56419038772583,
      "kl": 0.1650390625,
      "learning_rate": 1.3634859523979134e-06,
      "loss": 0.0016,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 360
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 356.4285888671875,
      "epoch": 0.22935196950444728,
      "grad_norm": 3.0232937335968018,
      "kl": 0.224609375,
      "learning_rate": 1.3520367982522208e-06,
      "loss": 0.0022,
      "reward": 1.1142858266830444,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.7142857313156128,
      "rewards/format_reward": 1.0,
      "step": 361
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 594.1428833007812,
      "epoch": 0.22998729351969505,
      "grad_norm": 3.2974321842193604,
      "kl": 0.1875,
      "learning_rate": 1.3406462998426358e-06,
      "loss": 0.0019,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 362
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 456.2857360839844,
      "epoch": 0.23062261753494281,
      "grad_norm": 3.04162859916687,
      "kl": 0.2119140625,
      "learning_rate": 1.3293149350916595e-06,
      "loss": 0.0021,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 363
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 532.4285888671875,
      "epoch": 0.2312579415501906,
      "grad_norm": 3.197749376296997,
      "kl": 0.1748046875,
      "learning_rate": 1.3180431794406623e-06,
      "loss": 0.0017,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 364
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 449.71429443359375,
      "epoch": 0.23189326556543838,
      "grad_norm": 3.0233240127563477,
      "kl": 0.2294921875,
      "learning_rate": 1.3068315058299358e-06,
      "loss": 0.0023,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 365
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 362.5714416503906,
      "epoch": 0.23252858958068615,
      "grad_norm": 0.06233400106430054,
      "kl": 0.28515625,
      "learning_rate": 1.2956803846788503e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 366
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 475.2857360839844,
      "epoch": 0.2331639135959339,
      "grad_norm": 0.04722089692950249,
      "kl": 0.193359375,
      "learning_rate": 1.284590283866116e-06,
      "loss": 0.0022,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 367
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 644.5714721679688,
      "epoch": 0.2337992376111817,
      "grad_norm": 2.4597692489624023,
      "kl": 0.265625,
      "learning_rate": 1.2735616687101518e-06,
      "loss": 0.0027,
      "reward": 0.4714285731315613,
      "reward_std": 0.18898221850395203,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0714285746216774,
      "rewards/format_reward": 1.0,
      "step": 368
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 353.8571472167969,
      "epoch": 0.23443456162642948,
      "grad_norm": 3.595822811126709,
      "kl": 0.255859375,
      "learning_rate": 1.2625950019495614e-06,
      "loss": 0.0026,
      "reward": 0.6106783151626587,
      "reward_std": 0.34680601954460144,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.21067820489406586,
      "rewards/format_reward": 1.0,
      "step": 369
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 585.5714721679688,
      "epoch": 0.23506988564167725,
      "grad_norm": 0.05966843292117119,
      "kl": 0.287109375,
      "learning_rate": 1.251690743723718e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 370
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 315.4285888671875,
      "epoch": 0.23570520965692504,
      "grad_norm": 3.2647478580474854,
      "kl": 0.26171875,
      "learning_rate": 1.2408493515534581e-06,
      "loss": 0.0026,
      "reward": 1.1142858266830444,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.7142857313156128,
      "rewards/format_reward": 1.0,
      "step": 371
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 503.0000305175781,
      "epoch": 0.2363405336721728,
      "grad_norm": 3.232701063156128,
      "kl": 0.1650390625,
      "learning_rate": 1.2300712803218834e-06,
      "loss": 0.0017,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 372
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 551.5714721679688,
      "epoch": 0.23697585768742058,
      "grad_norm": 0.05632919818162918,
      "kl": 0.2890625,
      "learning_rate": 1.2193569822552772e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 373
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 407.0000305175781,
      "epoch": 0.23761118170266837,
      "grad_norm": 0.06528866291046143,
      "kl": 0.228515625,
      "learning_rate": 1.2087069069041268e-06,
      "loss": 0.0026,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 374
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 378.8571472167969,
      "epoch": 0.23824650571791614,
      "grad_norm": 3.4633543491363525,
      "kl": 0.29296875,
      "learning_rate": 1.1981215011242654e-06,
      "loss": 0.0029,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 375
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 499.0000305175781,
      "epoch": 0.2388818297331639,
      "grad_norm": 0.06882923096418381,
      "kl": 0.2216796875,
      "learning_rate": 1.1876012090581184e-06,
      "loss": 0.0025,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 376
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 424.8571472167969,
      "epoch": 0.23951715374841168,
      "grad_norm": 0.048563580960035324,
      "kl": 0.1806640625,
      "learning_rate": 1.177146472116071e-06,
      "loss": 0.0021,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 377
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 556.1428833007812,
      "epoch": 0.24015247776365947,
      "grad_norm": 3.4067060947418213,
      "kl": 0.263671875,
      "learning_rate": 1.1667577289579462e-06,
      "loss": 0.0026,
      "reward": 0.43296706676483154,
      "reward_std": 0.06052277237176895,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.032967038452625275,
      "rewards/format_reward": 1.0,
      "step": 378
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 415.0000305175781,
      "epoch": 0.24078780177890724,
      "grad_norm": 2.6581125259399414,
      "kl": 0.3203125,
      "learning_rate": 1.1564354154746007e-06,
      "loss": 0.0032,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 379
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 473.0000305175781,
      "epoch": 0.241423125794155,
      "grad_norm": 0.12217168509960175,
      "kl": 0.2734375,
      "learning_rate": 1.146179964769635e-06,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 380
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 358.5714416503906,
      "epoch": 0.2420584498094028,
      "grad_norm": 0.07087297737598419,
      "kl": 0.337890625,
      "learning_rate": 1.1359918071412195e-06,
      "loss": 0.0037,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 381
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 666.7142944335938,
      "epoch": 0.24269377382465057,
      "grad_norm": 3.078494071960449,
      "kl": 0.1669921875,
      "learning_rate": 1.1258713700640456e-06,
      "loss": 0.0017,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 382
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 573.5714721679688,
      "epoch": 0.24332909783989834,
      "grad_norm": 0.0404311902821064,
      "kl": 0.25,
      "learning_rate": 1.115819078171383e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 383
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 508.857177734375,
      "epoch": 0.24396442185514614,
      "grad_norm": 2.985482931137085,
      "kl": 0.1787109375,
      "learning_rate": 1.1058353532372667e-06,
      "loss": 0.0018,
      "reward": 0.37460315227508545,
      "reward_std": 0.07708179205656052,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0031746032182127237,
      "rewards/format_reward": 1.0,
      "step": 384
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 499.857177734375,
      "epoch": 0.2445997458703939,
      "grad_norm": 2.4948058128356934,
      "kl": 0.251953125,
      "learning_rate": 1.0959206141587998e-06,
      "loss": 0.0025,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 385
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.2857360839844,
      "epoch": 0.24523506988564167,
      "grad_norm": 3.1324355602264404,
      "kl": 0.2119140625,
      "learning_rate": 1.0860752769385766e-06,
      "loss": 0.0021,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 386
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.14288330078125,
      "epoch": 0.24587039390088947,
      "grad_norm": 0.061597954481840134,
      "kl": 0.28515625,
      "learning_rate": 1.0762997546672279e-06,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 387
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 500.4285888671875,
      "epoch": 0.24650571791613723,
      "grad_norm": 0.047814078629016876,
      "kl": 0.2021484375,
      "learning_rate": 1.0665944575060914e-06,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 388
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 375.71429443359375,
      "epoch": 0.247141041931385,
      "grad_norm": 3.590963363647461,
      "kl": 0.22265625,
      "learning_rate": 1.056959792669997e-06,
      "loss": 0.0022,
      "reward": 0.5285714268684387,
      "reward_std": 0.11126972734928131,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.12857143580913544,
      "rewards/format_reward": 1.0,
      "step": 389
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 470.0000305175781,
      "epoch": 0.24777636594663277,
      "grad_norm": 0.10802248120307922,
      "kl": 0.376953125,
      "learning_rate": 1.0473961644101856e-06,
      "loss": 0.0041,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 390
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 475.14288330078125,
      "epoch": 0.24841168996188057,
      "grad_norm": 0.18473738431930542,
      "kl": 0.318359375,
      "learning_rate": 1.037903973997345e-06,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 391
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 455.5714416503906,
      "epoch": 0.24904701397712833,
      "grad_norm": 4.052010536193848,
      "kl": 0.349609375,
      "learning_rate": 1.0284836197047737e-06,
      "loss": 0.0035,
      "reward": 0.49629050493240356,
      "reward_std": 0.17879442870616913,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0962904542684555,
      "rewards/format_reward": 1.0,
      "step": 392
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 437.0000305175781,
      "epoch": 0.2496823379923761,
      "grad_norm": 2.7656362056732178,
      "kl": 0.2158203125,
      "learning_rate": 1.0191354967916712e-06,
      "loss": 0.0022,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 393
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 513.2857666015625,
      "epoch": 0.2503176620076239,
      "grad_norm": 0.08351174741983414,
      "kl": 0.275390625,
      "learning_rate": 1.0098599974865515e-06,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 394
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 429.4285888671875,
      "epoch": 0.25095298602287164,
      "grad_norm": 0.061210744082927704,
      "kl": 0.248046875,
      "learning_rate": 1.0006575109707898e-06,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 395
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 506.71429443359375,
      "epoch": 0.25158831003811943,
      "grad_norm": 0.06361155211925507,
      "kl": 0.2734375,
      "learning_rate": 9.915284233622877e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 396
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 431.4285888671875,
      "epoch": 0.25222363405336723,
      "grad_norm": 0.06987325847148895,
      "kl": 0.203125,
      "learning_rate": 9.824731176992796e-07,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 397
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 355.14288330078125,
      "epoch": 0.25285895806861497,
      "grad_norm": 0.1327361762523651,
      "kl": 0.30859375,
      "learning_rate": 9.734919739242543e-07,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 398
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 417.5714416503906,
      "epoch": 0.25349428208386277,
      "grad_norm": 3.1446924209594727,
      "kl": 0.333984375,
      "learning_rate": 9.645853688680177e-07,
      "loss": 0.0033,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 399
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 389.5714416503906,
      "epoch": 0.25412960609911056,
      "grad_norm": 0.048515211790800095,
      "kl": 0.1787109375,
      "learning_rate": 9.557536762338786e-07,
      "loss": 0.0021,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 400
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.14288330078125,
      "epoch": 0.2547649301143583,
      "grad_norm": 0.053919967263936996,
      "kl": 0.2109375,
      "learning_rate": 9.46997266581973e-07,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 401
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 408.2857360839844,
      "epoch": 0.2554002541296061,
      "grad_norm": 3.3761556148529053,
      "kl": 0.310546875,
      "learning_rate": 9.383165073137115e-07,
      "loss": 0.0031,
      "reward": 0.44395607709884644,
      "reward_std": 0.04111712798476219,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.04395604878664017,
      "rewards/format_reward": 1.0,
      "step": 402
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 423.8571472167969,
      "epoch": 0.2560355781448539,
      "grad_norm": 0.17396095395088196,
      "kl": 0.3515625,
      "learning_rate": 9.297117626563687e-07,
      "loss": 0.0038,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 403
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 589.7142944335938,
      "epoch": 0.25667090216010163,
      "grad_norm": 0.44405487179756165,
      "kl": 0.318359375,
      "learning_rate": 9.211833936477957e-07,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 404
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.0000305175781,
      "epoch": 0.25730622617534943,
      "grad_norm": 0.06627284735441208,
      "kl": 0.29296875,
      "learning_rate": 9.127317581212753e-07,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 405
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 381.0000305175781,
      "epoch": 0.2579415501905972,
      "grad_norm": 0.10693392902612686,
      "kl": 0.30078125,
      "learning_rate": 9.043572106905084e-07,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 406
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.0000305175781,
      "epoch": 0.25857687420584496,
      "grad_norm": 0.04814046248793602,
      "kl": 0.26953125,
      "learning_rate": 8.960601027347321e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 407
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 390.4285888671875,
      "epoch": 0.25921219822109276,
      "grad_norm": 0.06620350480079651,
      "kl": 0.21484375,
      "learning_rate": 8.878407823839788e-07,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 408
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 444.4285888671875,
      "epoch": 0.25984752223634056,
      "grad_norm": 3.463074207305908,
      "kl": 0.376953125,
      "learning_rate": 8.796995945044689e-07,
      "loss": 0.0038,
      "reward": 0.4129870533943176,
      "reward_std": 0.034360405057668686,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.012987013906240463,
      "rewards/format_reward": 1.0,
      "step": 409
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 442.5714416503906,
      "epoch": 0.2604828462515883,
      "grad_norm": 2.6474926471710205,
      "kl": 0.287109375,
      "learning_rate": 8.716368806841405e-07,
      "loss": 0.0029,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 410
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 393.4285888671875,
      "epoch": 0.2611181702668361,
      "grad_norm": 3.3637120723724365,
      "kl": 0.203125,
      "learning_rate": 8.636529792183171e-07,
      "loss": 0.002,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 411
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 441.857177734375,
      "epoch": 0.2617534942820839,
      "grad_norm": 2.904768466949463,
      "kl": 0.2060546875,
      "learning_rate": 8.557482250955144e-07,
      "loss": 0.0021,
      "reward": 0.6987013220787048,
      "reward_std": 0.21078045666217804,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.29870128631591797,
      "rewards/format_reward": 1.0,
      "step": 412
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 399.14288330078125,
      "epoch": 0.2623888182973316,
      "grad_norm": 0.10355106741189957,
      "kl": 0.3359375,
      "learning_rate": 8.479229499833844e-07,
      "loss": 0.0037,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 413
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 456.0000305175781,
      "epoch": 0.2630241423125794,
      "grad_norm": 0.13930782675743103,
      "kl": 0.294921875,
      "learning_rate": 8.401774822147976e-07,
      "loss": 0.0032,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 414
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 443.4285888671875,
      "epoch": 0.2636594663278272,
      "grad_norm": 3.3897175788879395,
      "kl": 0.306640625,
      "learning_rate": 8.325121467740695e-07,
      "loss": 0.0031,
      "reward": 0.5224490165710449,
      "reward_std": 0.12853363156318665,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.12244899570941925,
      "rewards/format_reward": 1.0,
      "step": 415
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 550.1428833007812,
      "epoch": 0.26429479034307496,
      "grad_norm": 0.05424835905432701,
      "kl": 0.25390625,
      "learning_rate": 8.249272652833226e-07,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 416
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 482.14288330078125,
      "epoch": 0.26493011435832275,
      "grad_norm": 0.2863621115684509,
      "kl": 0.3203125,
      "learning_rate": 8.174231559889931e-07,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 417
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 486.5714416503906,
      "epoch": 0.2655654383735705,
      "grad_norm": 0.05610418692231178,
      "kl": 0.28515625,
      "learning_rate": 8.100001337484787e-07,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 418
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.2857360839844,
      "epoch": 0.2662007623888183,
      "grad_norm": 0.07580099999904633,
      "kl": 0.322265625,
      "learning_rate": 8.026585100169251e-07,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 419
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 478.71429443359375,
      "epoch": 0.2668360864040661,
      "grad_norm": 3.3928205966949463,
      "kl": 0.23046875,
      "learning_rate": 7.953985928341601e-07,
      "loss": 0.0023,
      "reward": 0.4655141234397888,
      "reward_std": 0.08343012630939484,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.06551411002874374,
      "rewards/format_reward": 1.0,
      "step": 420
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 494.0000305175781,
      "epoch": 0.2674714104193138,
      "grad_norm": 0.04618528112769127,
      "kl": 0.1669921875,
      "learning_rate": 7.882206868117693e-07,
      "loss": 0.002,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 421
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 441.5714416503906,
      "epoch": 0.2681067344345616,
      "grad_norm": 2.9898011684417725,
      "kl": 0.28515625,
      "learning_rate": 7.81125093120313e-07,
      "loss": 0.0028,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 422
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 462.4285888671875,
      "epoch": 0.2687420584498094,
      "grad_norm": 0.056119803339242935,
      "kl": 0.3046875,
      "learning_rate": 7.741121094766916e-07,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 423
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 312.2857360839844,
      "epoch": 0.26937738246505716,
      "grad_norm": 3.943939447402954,
      "kl": 0.314453125,
      "learning_rate": 7.671820301316532e-07,
      "loss": 0.0031,
      "reward": 0.48671239614486694,
      "reward_std": 0.044492121785879135,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.08671241253614426,
      "rewards/format_reward": 1.0,
      "step": 424
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.4285888671875,
      "epoch": 0.27001270648030495,
      "grad_norm": 0.04282496124505997,
      "kl": 0.16015625,
      "learning_rate": 7.603351458574474e-07,
      "loss": 0.0019,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 425
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 496.2857360839844,
      "epoch": 0.27064803049555275,
      "grad_norm": 0.059300344437360764,
      "kl": 0.26953125,
      "learning_rate": 7.535717439356255e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 426
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.14288330078125,
      "epoch": 0.2712833545108005,
      "grad_norm": 2.4999890327453613,
      "kl": 0.14453125,
      "learning_rate": 7.46892108144986e-07,
      "loss": 0.0014,
      "reward": 0.7171429395675659,
      "reward_std": 0.4510992467403412,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.31714287400245667,
      "rewards/format_reward": 1.0,
      "step": 427
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 488.71429443359375,
      "epoch": 0.2719186785260483,
      "grad_norm": 0.052872247993946075,
      "kl": 0.279296875,
      "learning_rate": 7.402965187496697e-07,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 428
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 475.857177734375,
      "epoch": 0.2725540025412961,
      "grad_norm": 2.7971818447113037,
      "kl": 0.1826171875,
      "learning_rate": 7.337852524873974e-07,
      "loss": 0.0018,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 429
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 647.1428833007812,
      "epoch": 0.2731893265565438,
      "grad_norm": 2.0566720962524414,
      "kl": 0.2255859375,
      "learning_rate": 7.273585825578608e-07,
      "loss": 0.0023,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 430
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.8571472167969,
      "epoch": 0.2738246505717916,
      "grad_norm": 2.9699957370758057,
      "kl": 0.27734375,
      "learning_rate": 7.21016778611259e-07,
      "loss": 0.0028,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 431
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.14288330078125,
      "epoch": 0.2744599745870394,
      "grad_norm": 0.10036282241344452,
      "kl": 0.31640625,
      "learning_rate": 7.147601067369835e-07,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 432
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 323.71429443359375,
      "epoch": 0.27509529860228715,
      "grad_norm": 3.8502118587493896,
      "kl": 0.275390625,
      "learning_rate": 7.085888294524561e-07,
      "loss": 0.0028,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 433
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 631.2857666015625,
      "epoch": 0.27573062261753495,
      "grad_norm": 0.05429592728614807,
      "kl": 0.23828125,
      "learning_rate": 7.025032056921117e-07,
      "loss": 0.0027,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 434
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 455.71429443359375,
      "epoch": 0.27636594663278274,
      "grad_norm": 3.2364959716796875,
      "kl": 0.29296875,
      "learning_rate": 6.965034907965349e-07,
      "loss": 0.0029,
      "reward": 0.5190476179122925,
      "reward_std": 0.09449110925197601,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1190476268529892,
      "rewards/format_reward": 1.0,
      "step": 435
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.0000305175781,
      "epoch": 0.2770012706480305,
      "grad_norm": 3.7509405612945557,
      "kl": 0.1748046875,
      "learning_rate": 6.905899365017462e-07,
      "loss": 0.0017,
      "reward": 0.4330357313156128,
      "reward_std": 0.0516289584338665,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.03303571790456772,
      "rewards/format_reward": 1.0,
      "step": 436
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 386.5714416503906,
      "epoch": 0.2776365946632783,
      "grad_norm": 0.05398930609226227,
      "kl": 0.314453125,
      "learning_rate": 6.847627909286409e-07,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 437
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 474.4285888671875,
      "epoch": 0.2782719186785261,
      "grad_norm": 2.960287094116211,
      "kl": 0.181640625,
      "learning_rate": 6.790222985725761e-07,
      "loss": 0.0018,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 438
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 492.4285888671875,
      "epoch": 0.2789072426937738,
      "grad_norm": 3.587923288345337,
      "kl": 0.2470703125,
      "learning_rate": 6.733687002931141e-07,
      "loss": 0.0025,
      "reward": 0.4714285731315613,
      "reward_std": 0.18898221850395203,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0714285746216774,
      "rewards/format_reward": 1.0,
      "step": 439
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.5714416503906,
      "epoch": 0.2795425667090216,
      "grad_norm": 2.850022077560425,
      "kl": 0.1650390625,
      "learning_rate": 6.678022333039158e-07,
      "loss": 0.0016,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 440
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.4285888671875,
      "epoch": 0.28017789072426935,
      "grad_norm": 0.045945875346660614,
      "kl": 0.265625,
      "learning_rate": 6.623231311627876e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 441
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 554.5714721679688,
      "epoch": 0.28081321473951715,
      "grad_norm": 0.06309759616851807,
      "kl": 0.265625,
      "learning_rate": 6.569316237618811e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 442
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 318.0,
      "epoch": 0.28144853875476494,
      "grad_norm": 3.4273765087127686,
      "kl": 0.234375,
      "learning_rate": 6.516279373180499e-07,
      "loss": 0.0023,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 443
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 551.4285888671875,
      "epoch": 0.2820838627700127,
      "grad_norm": 0.059127844870090485,
      "kl": 0.2490234375,
      "learning_rate": 6.464122943633543e-07,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 444
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 388.71429443359375,
      "epoch": 0.2827191867852605,
      "grad_norm": 0.058766093105077744,
      "kl": 0.302734375,
      "learning_rate": 6.412849137357271e-07,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 445
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 484.2857360839844,
      "epoch": 0.2833545108005083,
      "grad_norm": 0.05756891146302223,
      "kl": 0.275390625,
      "learning_rate": 6.3624601056979e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 446
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 397.4285888671875,
      "epoch": 0.283989834815756,
      "grad_norm": 3.6736083030700684,
      "kl": 0.30859375,
      "learning_rate": 6.312957962878278e-07,
      "loss": 0.0031,
      "reward": 0.4039139151573181,
      "reward_std": 0.01035518478602171,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.003913894295692444,
      "rewards/format_reward": 1.0,
      "step": 447
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 528.857177734375,
      "epoch": 0.2846251588310038,
      "grad_norm": 0.047244369983673096,
      "kl": 0.259765625,
      "learning_rate": 6.264344785909181e-07,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 448
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 432.5714416503906,
      "epoch": 0.2852604828462516,
      "grad_norm": 0.06841956079006195,
      "kl": 0.298828125,
      "learning_rate": 6.216622614502149e-07,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 449
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 538.1428833007812,
      "epoch": 0.28589580686149935,
      "grad_norm": 0.052464455366134644,
      "kl": 0.25,
      "learning_rate": 6.169793450983916e-07,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 450
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 490.71429443359375,
      "epoch": 0.28653113087674714,
      "grad_norm": 0.04570423439145088,
      "kl": 0.2470703125,
      "learning_rate": 6.123859260212393e-07,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 451
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 400.14288330078125,
      "epoch": 0.28716645489199494,
      "grad_norm": 0.05332702025771141,
      "kl": 0.306640625,
      "learning_rate": 6.07882196949423e-07,
      "loss": 0.0034,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 452
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 338.8571472167969,
      "epoch": 0.2878017789072427,
      "grad_norm": 3.625446081161499,
      "kl": 0.1982421875,
      "learning_rate": 6.034683468503948e-07,
      "loss": 0.002,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 453
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 361.14288330078125,
      "epoch": 0.2884371029224905,
      "grad_norm": 3.3959381580352783,
      "kl": 0.375,
      "learning_rate": 5.991445609204641e-07,
      "loss": 0.0037,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 454
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 404.2857360839844,
      "epoch": 0.28907242693773827,
      "grad_norm": 0.06705067306756973,
      "kl": 0.27734375,
      "learning_rate": 5.949110205770292e-07,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 455
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 368.14288330078125,
      "epoch": 0.289707750952986,
      "grad_norm": 0.0792422741651535,
      "kl": 0.30078125,
      "learning_rate": 5.90767903450964e-07,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 456
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 356.0000305175781,
      "epoch": 0.2903430749682338,
      "grad_norm": 3.4945454597473145,
      "kl": 0.1982421875,
      "learning_rate": 5.867153833791652e-07,
      "loss": 0.002,
      "reward": 0.4571428596973419,
      "reward_std": 0.053452249616384506,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.05714286118745804,
      "rewards/format_reward": 1.0,
      "step": 457
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 371.5714416503906,
      "epoch": 0.2909783989834816,
      "grad_norm": 3.341874837875366,
      "kl": 0.3203125,
      "learning_rate": 5.827536303972587e-07,
      "loss": 0.0032,
      "reward": 0.44761908054351807,
      "reward_std": 0.12598817050457,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0476190522313118,
      "rewards/format_reward": 1.0,
      "step": 458
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.8571472167969,
      "epoch": 0.29161372299872934,
      "grad_norm": 0.0777309313416481,
      "kl": 0.30078125,
      "learning_rate": 5.78882810732465e-07,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 459
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 461.857177734375,
      "epoch": 0.29224904701397714,
      "grad_norm": 0.0932033360004425,
      "kl": 0.3203125,
      "learning_rate": 5.75103086796625e-07,
      "loss": 0.0035,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 460
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 419.4285888671875,
      "epoch": 0.2928843710292249,
      "grad_norm": 3.0678720474243164,
      "kl": 0.1650390625,
      "learning_rate": 5.714146171793846e-07,
      "loss": 0.0016,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 461
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.71429443359375,
      "epoch": 0.2935196950444727,
      "grad_norm": 0.08228779584169388,
      "kl": 0.216796875,
      "learning_rate": 5.678175566415422e-07,
      "loss": 0.0025,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 462
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 443.71429443359375,
      "epoch": 0.29415501905972047,
      "grad_norm": 2.509739398956299,
      "kl": 0.349609375,
      "learning_rate": 5.643120561085528e-07,
      "loss": 0.0035,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645264148712,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 463
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 476.857177734375,
      "epoch": 0.2947903430749682,
      "grad_norm": 0.0598638616502285,
      "kl": 0.26171875,
      "learning_rate": 5.608982626641991e-07,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 464
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 508.0000305175781,
      "epoch": 0.295425667090216,
      "grad_norm": 0.04956334829330444,
      "kl": 0.2109375,
      "learning_rate": 5.575763195444166e-07,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 465
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 519.1428833007812,
      "epoch": 0.2960609911054638,
      "grad_norm": 3.209406614303589,
      "kl": 0.275390625,
      "learning_rate": 5.543463661312847e-07,
      "loss": 0.0027,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 466
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 440.857177734375,
      "epoch": 0.29669631512071154,
      "grad_norm": 0.05284934490919113,
      "kl": 0.26171875,
      "learning_rate": 5.512085379471808e-07,
      "loss": 0.0029,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 467
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 445.2857360839844,
      "epoch": 0.29733163913595934,
      "grad_norm": 3.3668439388275146,
      "kl": 0.29296875,
      "learning_rate": 5.481629666490903e-07,
      "loss": 0.0029,
      "reward": 0.485714316368103,
      "reward_std": 0.08997353911399841,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.08571429550647736,
      "rewards/format_reward": 1.0,
      "step": 468
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 541.0,
      "epoch": 0.29796696315120713,
      "grad_norm": 2.8946890830993652,
      "kl": 0.2216796875,
      "learning_rate": 5.452097800230853e-07,
      "loss": 0.0022,
      "reward": 0.3428571820259094,
      "reward_std": 0.1511857956647873,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 469
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 425.4285888671875,
      "epoch": 0.29860228716645487,
      "grad_norm": 0.04910692945122719,
      "kl": 0.1669921875,
      "learning_rate": 5.423491019789623e-07,
      "loss": 0.002,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 470
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 238.2857208251953,
      "epoch": 0.29923761118170267,
      "grad_norm": 0.06361490488052368,
      "kl": 0.208984375,
      "learning_rate": 5.395810525450425e-07,
      "loss": 0.0033,
      "reward": 1.4000002145767212,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 1.0,
      "rewards/format_reward": 1.0,
      "step": 471
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 395.8571472167969,
      "epoch": 0.29987293519695046,
      "grad_norm": 0.04494641348719597,
      "kl": 0.2041015625,
      "learning_rate": 5.369057478631359e-07,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 472
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 438.0000305175781,
      "epoch": 0.3005082592121982,
      "grad_norm": 3.899231195449829,
      "kl": 0.2099609375,
      "learning_rate": 5.343233001836694e-07,
      "loss": 0.0021,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 473
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 566.1428833007812,
      "epoch": 0.301143583227446,
      "grad_norm": 2.999528646469116,
      "kl": 0.1689453125,
      "learning_rate": 5.318338178609754e-07,
      "loss": 0.0017,
      "reward": 0.42418450117111206,
      "reward_std": 0.06398611515760422,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.02418447658419609,
      "rewards/format_reward": 1.0,
      "step": 474
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 505.2857360839844,
      "epoch": 0.3017789072426938,
      "grad_norm": 0.056534409523010254,
      "kl": 0.2138671875,
      "learning_rate": 5.294374053487459e-07,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 475
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 376.71429443359375,
      "epoch": 0.30241423125794153,
      "grad_norm": 3.721620798110962,
      "kl": 0.212890625,
      "learning_rate": 5.271341631956511e-07,
      "loss": 0.0021,
      "reward": 0.5836734771728516,
      "reward_std": 0.3662114441394806,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.18367348611354828,
      "rewards/format_reward": 1.0,
      "step": 476
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 492.14288330078125,
      "epoch": 0.30304955527318933,
      "grad_norm": 0.05311813950538635,
      "kl": 0.251953125,
      "learning_rate": 5.249241880411181e-07,
      "loss": 0.0028,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 477
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 510.0000305175781,
      "epoch": 0.3036848792884371,
      "grad_norm": 3.715238332748413,
      "kl": 0.240234375,
      "learning_rate": 5.228075726112785e-07,
      "loss": 0.0024,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645264148712,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 478
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 455.0000305175781,
      "epoch": 0.30432020330368487,
      "grad_norm": 3.2647809982299805,
      "kl": 0.1640625,
      "learning_rate": 5.207844057150768e-07,
      "loss": 0.0016,
      "reward": 0.5428571701049805,
      "reward_std": 0.37796446681022644,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 479
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 481.71429443359375,
      "epoch": 0.30495552731893266,
      "grad_norm": 0.06345637142658234,
      "kl": 0.265625,
      "learning_rate": 5.188547722405437e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 480
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 439.14288330078125,
      "epoch": 0.30559085133418046,
      "grad_norm": 0.06628384441137314,
      "kl": 0.26953125,
      "learning_rate": 5.170187531512351e-07,
      "loss": 0.003,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 481
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.14288330078125,
      "epoch": 0.3062261753494282,
      "grad_norm": 2.7409112453460693,
      "kl": 0.1591796875,
      "learning_rate": 5.152764254828348e-07,
      "loss": 0.0016,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645562171936,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 482
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 573.1428833007812,
      "epoch": 0.306861499364676,
      "grad_norm": 0.041587937623262405,
      "kl": 0.140625,
      "learning_rate": 5.136278623399225e-07,
      "loss": 0.0017,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 483
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 375.0000305175781,
      "epoch": 0.30749682337992373,
      "grad_norm": 2.9096410274505615,
      "kl": 0.2001953125,
      "learning_rate": 5.120731328929058e-07,
      "loss": 0.002,
      "reward": 0.971428632736206,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.5714285969734192,
      "rewards/format_reward": 1.0,
      "step": 484
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 454.0000305175781,
      "epoch": 0.30813214739517153,
      "grad_norm": 3.31744122505188,
      "kl": 0.21484375,
      "learning_rate": 5.106123023751187e-07,
      "loss": 0.0021,
      "reward": 0.8511278033256531,
      "reward_std": 0.23666103184223175,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.451127827167511,
      "rewards/format_reward": 1.0,
      "step": 485
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 416.4285888671875,
      "epoch": 0.3087674714104193,
      "grad_norm": 2.8466956615448,
      "kl": 0.19140625,
      "learning_rate": 5.092454320800833e-07,
      "loss": 0.0019,
      "reward": 0.4338059425354004,
      "reward_std": 0.06710861623287201,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.03380589187145233,
      "rewards/format_reward": 1.0,
      "step": 486
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 466.4285888671875,
      "epoch": 0.30940279542566707,
      "grad_norm": 5.7825751304626465,
      "kl": 0.65234375,
      "learning_rate": 5.079725793589405e-07,
      "loss": 0.0068,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 487
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 358.4285888671875,
      "epoch": 0.31003811944091486,
      "grad_norm": 0.06072893738746643,
      "kl": 0.1953125,
      "learning_rate": 5.067937976180407e-07,
      "loss": 0.0022,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 488
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 504.2857360839844,
      "epoch": 0.31067344345616266,
      "grad_norm": 0.03951825201511383,
      "kl": 0.1630859375,
      "learning_rate": 5.057091363167046e-07,
      "loss": 0.0019,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 489
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 235.1428680419922,
      "epoch": 0.3113087674714104,
      "grad_norm": 3.270069122314453,
      "kl": 0.224609375,
      "learning_rate": 5.047186409651489e-07,
      "loss": 0.0022,
      "reward": 1.3428572416305542,
      "reward_std": 0.15118582546710968,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 1.0,
      "rewards/format_reward": 0.8571429252624512,
      "step": 490
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 509.0000305175781,
      "epoch": 0.3119440914866582,
      "grad_norm": 0.1327732652425766,
      "kl": 0.298828125,
      "learning_rate": 5.038223531225742e-07,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 491
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 507.2857360839844,
      "epoch": 0.312579415501906,
      "grad_norm": 3.286302328109741,
      "kl": 0.2353515625,
      "learning_rate": 5.030203103954232e-07,
      "loss": 0.0024,
      "reward": 0.37142860889434814,
      "reward_std": 0.07559289783239365,
      "rewards/code_format_reward": 0.8571429252624512,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 492
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 433.71429443359375,
      "epoch": 0.31321473951715373,
      "grad_norm": 0.04749950394034386,
      "kl": 0.173828125,
      "learning_rate": 5.023125464358026e-07,
      "loss": 0.002,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 493
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 402.2857360839844,
      "epoch": 0.3138500635324015,
      "grad_norm": 3.375169515609741,
      "kl": 0.26171875,
      "learning_rate": 5.016990909400709e-07,
      "loss": 0.0026,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 494
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 491.857177734375,
      "epoch": 0.3144853875476493,
      "grad_norm": 0.05139714851975441,
      "kl": 0.296875,
      "learning_rate": 5.011799696475915e-07,
      "loss": 0.0033,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 495
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 323.5714416503906,
      "epoch": 0.31512071156289706,
      "grad_norm": 3.7574667930603027,
      "kl": 0.181640625,
      "learning_rate": 5.007552043396547e-07,
      "loss": 0.0018,
      "reward": 0.5428571701049805,
      "reward_std": 0.3779645264148712,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.1428571492433548,
      "rewards/format_reward": 1.0,
      "step": 496
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 534.857177734375,
      "epoch": 0.31575603557814486,
      "grad_norm": 2.853102445602417,
      "kl": 0.1806640625,
      "learning_rate": 5.004248128385618e-07,
      "loss": 0.0018,
      "reward": 0.6857143640518188,
      "reward_std": 0.4879501163959503,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.2857142984867096,
      "rewards/format_reward": 1.0,
      "step": 497
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 422.8571472167969,
      "epoch": 0.31639135959339265,
      "grad_norm": 3.089301824569702,
      "kl": 0.1923828125,
      "learning_rate": 5.001888090068784e-07,
      "loss": 0.0019,
      "reward": 0.8285714983940125,
      "reward_std": 0.534522533416748,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.4285714626312256,
      "rewards/format_reward": 1.0,
      "step": 498
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 451.2857360839844,
      "epoch": 0.3170266836086404,
      "grad_norm": 5.161578178405762,
      "kl": 0.57421875,
      "learning_rate": 5.000472027468528e-07,
      "loss": 0.0057,
      "reward": 0.2857142984867096,
      "reward_std": 0.1951800137758255,
      "rewards/code_format_reward": 0.7142857313156128,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 0.7142857313156128,
      "step": 499
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 611.4285888671875,
      "epoch": 0.3176620076238882,
      "grad_norm": 0.048863135278224945,
      "kl": 0.2060546875,
      "learning_rate": 5.000000000000001e-07,
      "loss": 0.0024,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 500
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 444.857177734375,
      "epoch": 0.318297331639136,
      "grad_norm": 0.047799259424209595,
      "kl": 0.19921875,
      "learning_rate": 5.000472027468528e-07,
      "loss": 0.0023,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 501
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 446.5714416503906,
      "epoch": 0.3189326556543837,
      "grad_norm": 0.15391913056373596,
      "kl": 0.283203125,
      "learning_rate": 5.001888090068784e-07,
      "loss": 0.0031,
      "reward": 0.40000003576278687,
      "reward_std": 0.0,
      "rewards/code_format_reward": 1.0,
      "rewards/code_reward": 0.0,
      "rewards/format_reward": 1.0,
      "step": 502
    },
    {
      "epoch": 0.3189326556543837,
      "step": 502,
      "total_flos": 0.0,
      "train_loss": 6.229176085843033e-06,
      "train_runtime": 154.2548,
      "train_samples_per_second": 22.69,
      "train_steps_per_second": 3.241
    }
  ],
  "logging_steps": 1,
  "max_steps": 500,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}