{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 500,
  "global_step": 375,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 270.8035888671875,
      "epoch": 0.0026666666666666666,
      "grad_norm": 0.7334129323891274,
      "kl": 0.0001461505889892578,
      "learning_rate": 2e-06,
      "loss": 0.0712,
      "reward": 0.6089551746845245,
      "reward_std": 0.1534397415816784,
      "rewards/length_reward": 0.06294643878936768,
      "rewards/similarity_reward": 0.546008750796318,
      "step": 1
    },
    {
      "completion_length": 248.49555206298828,
      "epoch": 0.005333333333333333,
      "grad_norm": 0.9035152344267738,
      "kl": 0.00016069412231445312,
      "learning_rate": 2e-06,
      "loss": 0.06,
      "reward": 0.5894998908042908,
      "reward_std": 0.17237309366464615,
      "rewards/length_reward": 0.061160728335380554,
      "rewards/similarity_reward": 0.528339147567749,
      "step": 2
    },
    {
      "completion_length": 271.5848388671875,
      "epoch": 0.008,
      "grad_norm": 0.8948061881977104,
      "kl": 0.00017070770263671875,
      "learning_rate": 2e-06,
      "loss": 0.0442,
      "reward": 0.6276047825813293,
      "reward_std": 0.15094101428985596,
      "rewards/length_reward": 0.071428582072258,
      "rewards/similarity_reward": 0.5561762154102325,
      "step": 3
    },
    {
      "completion_length": 257.46429443359375,
      "epoch": 0.010666666666666666,
      "grad_norm": 0.7848935176364024,
      "kl": 0.00016450881958007812,
      "learning_rate": 2e-06,
      "loss": 0.0583,
      "reward": 0.5882956981658936,
      "reward_std": 0.1704816073179245,
      "rewards/length_reward": 0.06473215110599995,
      "rewards/similarity_reward": 0.5235635340213776,
      "step": 4
    },
    {
      "completion_length": 240.4821548461914,
      "epoch": 0.013333333333333334,
      "grad_norm": 0.7884926165963533,
      "kl": 0.000179290771484375,
      "learning_rate": 2e-06,
      "loss": 0.0444,
      "reward": 0.6010282635688782,
      "reward_std": 0.10747816786170006,
      "rewards/length_reward": 0.058482151478528976,
      "rewards/similarity_reward": 0.5425460636615753,
      "step": 5
    },
    {
      "completion_length": 271.5758972167969,
      "epoch": 0.016,
      "grad_norm": 0.7176003677981444,
      "kl": 0.0001480579376220703,
      "learning_rate": 2e-06,
      "loss": 0.1772,
      "reward": 0.632064938545227,
      "reward_std": 0.13313144445419312,
      "rewards/length_reward": 0.06562501192092896,
      "rewards/similarity_reward": 0.5664399564266205,
      "step": 6
    },
    {
      "completion_length": 248.83483123779297,
      "epoch": 0.018666666666666668,
      "grad_norm": 0.583690210979672,
      "kl": 0.000118255615234375,
      "learning_rate": 2e-06,
      "loss": 0.01,
      "reward": 0.6557432115077972,
      "reward_std": 0.09923132508993149,
      "rewards/length_reward": 0.0714285783469677,
      "rewards/similarity_reward": 0.584314614534378,
      "step": 7
    },
    {
      "completion_length": 240.2991180419922,
      "epoch": 0.021333333333333333,
      "grad_norm": 0.8285932775348754,
      "kl": 0.00018167495727539062,
      "learning_rate": 2e-06,
      "loss": 0.0352,
      "reward": 0.5436868965625763,
      "reward_std": 0.19458478689193726,
      "rewards/length_reward": 0.053571440279483795,
      "rewards/similarity_reward": 0.4901154637336731,
      "step": 8
    },
    {
      "completion_length": 274.74554443359375,
      "epoch": 0.024,
      "grad_norm": 0.6297183637280267,
      "kl": 0.00014972686767578125,
      "learning_rate": 2e-06,
      "loss": 0.0198,
      "reward": 0.6230277419090271,
      "reward_std": 0.15007304400205612,
      "rewards/length_reward": 0.06562501192092896,
      "rewards/similarity_reward": 0.5574026703834534,
      "step": 9
    },
    {
      "completion_length": 271.2991180419922,
      "epoch": 0.02666666666666667,
      "grad_norm": 0.6908976518587052,
      "kl": 0.00015354156494140625,
      "learning_rate": 2e-06,
      "loss": 0.0383,
      "reward": 0.6355253756046295,
      "reward_std": 0.16409822553396225,
      "rewards/length_reward": 0.0647321492433548,
      "rewards/similarity_reward": 0.5707932114601135,
      "step": 10
    },
    {
      "completion_length": 263.7901916503906,
      "epoch": 0.029333333333333333,
      "grad_norm": 0.7190922283358612,
      "kl": 0.00019359588623046875,
      "learning_rate": 2e-06,
      "loss": 0.039,
      "reward": 0.6213563680648804,
      "reward_std": 0.14211475104093552,
      "rewards/length_reward": 0.07053572311997414,
      "rewards/similarity_reward": 0.5508206188678741,
      "step": 11
    },
    {
      "completion_length": 268.21429443359375,
      "epoch": 0.032,
      "grad_norm": 0.7674287367860577,
      "kl": 0.0001621246337890625,
      "learning_rate": 2e-06,
      "loss": 0.0666,
      "reward": 0.6959913372993469,
      "reward_std": 0.13710426539182663,
      "rewards/length_reward": 0.07098215445876122,
      "rewards/similarity_reward": 0.6250091791152954,
      "step": 12
    },
    {
      "completion_length": 308.23216247558594,
      "epoch": 0.034666666666666665,
      "grad_norm": 0.6961227618622208,
      "kl": 0.00017309188842773438,
      "learning_rate": 2e-06,
      "loss": 0.1339,
      "reward": 0.6554811000823975,
      "reward_std": 0.14746695756912231,
      "rewards/length_reward": 0.06383929960429668,
      "rewards/similarity_reward": 0.5916417837142944,
      "step": 13
    },
    {
      "completion_length": 255.1696548461914,
      "epoch": 0.037333333333333336,
      "grad_norm": 0.6790296137610172,
      "kl": 0.00018310546875,
      "learning_rate": 2e-06,
      "loss": 0.0102,
      "reward": 0.639388918876648,
      "reward_std": 0.13504448905587196,
      "rewards/length_reward": 0.07232143729925156,
      "rewards/similarity_reward": 0.5670674443244934,
      "step": 14
    },
    {
      "completion_length": 288.5982208251953,
      "epoch": 0.04,
      "grad_norm": 0.5592059952702082,
      "kl": 0.00015926361083984375,
      "learning_rate": 2e-06,
      "loss": 0.0083,
      "reward": 0.6401915550231934,
      "reward_std": 0.10427659377455711,
      "rewards/length_reward": 0.061607155948877335,
      "rewards/similarity_reward": 0.5785843580961227,
      "step": 15
    },
    {
      "completion_length": 256.5669708251953,
      "epoch": 0.042666666666666665,
      "grad_norm": 0.6493017975612209,
      "kl": 0.0001373291015625,
      "learning_rate": 2e-06,
      "loss": 0.0011,
      "reward": 0.6510922610759735,
      "reward_std": 0.12598292529582977,
      "rewards/length_reward": 0.0669642984867096,
      "rewards/similarity_reward": 0.5841279327869415,
      "step": 16
    },
    {
      "completion_length": 306.2857208251953,
      "epoch": 0.04533333333333334,
      "grad_norm": 0.6883394230623637,
      "kl": 0.00019121170043945312,
      "learning_rate": 2e-06,
      "loss": 0.0685,
      "reward": 0.5360667556524277,
      "reward_std": 0.17403440922498703,
      "rewards/length_reward": 0.04866072162985802,
      "rewards/similarity_reward": 0.48740604519844055,
      "step": 17
    },
    {
      "completion_length": 332.41966247558594,
      "epoch": 0.048,
      "grad_norm": 0.5705875740256017,
      "kl": 0.00019073486328125,
      "learning_rate": 2e-06,
      "loss": 0.0755,
      "reward": 0.6520929932594299,
      "reward_std": 0.11369618400931358,
      "rewards/length_reward": 0.06428572162985802,
      "rewards/similarity_reward": 0.5878072679042816,
      "step": 18
    },
    {
      "completion_length": 285.03126525878906,
      "epoch": 0.050666666666666665,
      "grad_norm": 0.6938225330516384,
      "kl": 0.00016689300537109375,
      "learning_rate": 2e-06,
      "loss": 0.1144,
      "reward": 0.6287901103496552,
      "reward_std": 0.14219095557928085,
      "rewards/length_reward": 0.06071429327130318,
      "rewards/similarity_reward": 0.5680757761001587,
      "step": 19
    },
    {
      "completion_length": 271.70982360839844,
      "epoch": 0.05333333333333334,
      "grad_norm": 0.7297528927947673,
      "kl": 0.00022459030151367188,
      "learning_rate": 2e-06,
      "loss": 0.0185,
      "reward": 0.6177776157855988,
      "reward_std": 0.16364814341068268,
      "rewards/length_reward": 0.058482155203819275,
      "rewards/similarity_reward": 0.5592954754829407,
      "step": 20
    },
    {
      "completion_length": 288.85716247558594,
      "epoch": 0.056,
      "grad_norm": 0.7214386901249736,
      "kl": 0.0002155303955078125,
      "learning_rate": 2e-06,
      "loss": 0.0434,
      "reward": 0.597628653049469,
      "reward_std": 0.14737623929977417,
      "rewards/length_reward": 0.06205358728766441,
      "rewards/similarity_reward": 0.5355750769376755,
      "step": 21
    },
    {
      "completion_length": 286.6875228881836,
      "epoch": 0.058666666666666666,
      "grad_norm": 0.6632688424489647,
      "kl": 0.0001938343048095703,
      "learning_rate": 2e-06,
      "loss": 0.0844,
      "reward": 0.6639349460601807,
      "reward_std": 0.13361597061157227,
      "rewards/length_reward": 0.06205357797443867,
      "rewards/similarity_reward": 0.6018813252449036,
      "step": 22
    },
    {
      "completion_length": 274.4419708251953,
      "epoch": 0.06133333333333333,
      "grad_norm": 0.6481364691610889,
      "kl": 0.00017881393432617188,
      "learning_rate": 2e-06,
      "loss": -0.0025,
      "reward": 0.6752828657627106,
      "reward_std": 0.1561322584748268,
      "rewards/length_reward": 0.0625000149011612,
      "rewards/similarity_reward": 0.612782746553421,
      "step": 23
    },
    {
      "completion_length": 286.56697845458984,
      "epoch": 0.064,
      "grad_norm": 0.6864769985687215,
      "kl": 0.00017595291137695312,
      "learning_rate": 2e-06,
      "loss": 0.0716,
      "reward": 0.5987082123756409,
      "reward_std": 0.1384410411119461,
      "rewards/length_reward": 0.06428572349250317,
      "rewards/similarity_reward": 0.5344224572181702,
      "step": 24
    },
    {
      "completion_length": 279.7053680419922,
      "epoch": 0.06666666666666667,
      "grad_norm": 0.6989530874534239,
      "kl": 0.0001468658447265625,
      "learning_rate": 2e-06,
      "loss": 0.1417,
      "reward": 0.7139560580253601,
      "reward_std": 0.12727026268839836,
      "rewards/length_reward": 0.0691964402794838,
      "rewards/similarity_reward": 0.6447596251964569,
      "step": 25
    },
    {
      "completion_length": 269.00000762939453,
      "epoch": 0.06933333333333333,
      "grad_norm": 0.8072710062956403,
      "kl": 0.00020742416381835938,
      "learning_rate": 2e-06,
      "loss": 0.1312,
      "reward": 0.5711838901042938,
      "reward_std": 0.16566643118858337,
      "rewards/length_reward": 0.055803582072257996,
      "rewards/similarity_reward": 0.515380322933197,
      "step": 26
    },
    {
      "completion_length": 261.1339340209961,
      "epoch": 0.072,
      "grad_norm": 0.6813535460039397,
      "kl": 0.00016641616821289062,
      "learning_rate": 2e-06,
      "loss": 0.0717,
      "reward": 0.6621068716049194,
      "reward_std": 0.126276895403862,
      "rewards/length_reward": 0.0669642984867096,
      "rewards/similarity_reward": 0.5951426327228546,
      "step": 27
    },
    {
      "completion_length": 301.8169708251953,
      "epoch": 0.07466666666666667,
      "grad_norm": 0.7408476748329949,
      "kl": 0.00023174285888671875,
      "learning_rate": 2e-06,
      "loss": 0.0763,
      "reward": 0.5747545063495636,
      "reward_std": 0.13357429951429367,
      "rewards/length_reward": 0.05178572237491608,
      "rewards/similarity_reward": 0.5229687094688416,
      "step": 28
    },
    {
      "completion_length": 292.96876525878906,
      "epoch": 0.07733333333333334,
      "grad_norm": 0.6666274319697456,
      "kl": 0.0001964569091796875,
      "learning_rate": 2e-06,
      "loss": 0.1043,
      "reward": 0.6386013031005859,
      "reward_std": 0.12155063822865486,
      "rewards/length_reward": 0.0669642984867096,
      "rewards/similarity_reward": 0.5716370046138763,
      "step": 29
    },
    {
      "completion_length": 295.7589416503906,
      "epoch": 0.08,
      "grad_norm": 0.6572218858197006,
      "kl": 0.0002288818359375,
      "learning_rate": 2e-06,
      "loss": 0.0662,
      "reward": 0.6807651817798615,
      "reward_std": 0.16092021018266678,
      "rewards/length_reward": 0.0691964328289032,
      "rewards/similarity_reward": 0.6115686893463135,
      "step": 30
    },
    {
      "completion_length": 307.6383972167969,
      "epoch": 0.08266666666666667,
      "grad_norm": 0.7925535842055806,
      "kl": 0.00022029876708984375,
      "learning_rate": 2e-06,
      "loss": 0.2064,
      "reward": 0.5656463354825974,
      "reward_std": 0.1584705486893654,
      "rewards/length_reward": 0.051785726100206375,
      "rewards/similarity_reward": 0.5138606131076813,
      "step": 31
    },
    {
      "completion_length": 232.94197845458984,
      "epoch": 0.08533333333333333,
      "grad_norm": 0.7532498555139676,
      "kl": 0.00029754638671875,
      "learning_rate": 2e-06,
      "loss": 0.0905,
      "reward": 0.6083326637744904,
      "reward_std": 0.14686080068349838,
      "rewards/length_reward": 0.06339287012815475,
      "rewards/similarity_reward": 0.544939786195755,
      "step": 32
    },
    {
      "completion_length": 282.5089416503906,
      "epoch": 0.088,
      "grad_norm": 0.8072212411776664,
      "kl": 0.0002989768981933594,
      "learning_rate": 2e-06,
      "loss": 0.0575,
      "reward": 0.6441718935966492,
      "reward_std": 0.14405860379338264,
      "rewards/length_reward": 0.06919643096625805,
      "rewards/similarity_reward": 0.574975460767746,
      "step": 33
    },
    {
      "completion_length": 255.03125762939453,
      "epoch": 0.09066666666666667,
      "grad_norm": 0.7214716728436158,
      "kl": 0.00023508071899414062,
      "learning_rate": 2e-06,
      "loss": 0.0549,
      "reward": 0.65805384516716,
      "reward_std": 0.10812044516205788,
      "rewards/length_reward": 0.054017869755625725,
      "rewards/similarity_reward": 0.6040360331535339,
      "step": 34
    },
    {
      "completion_length": 318.4464416503906,
      "epoch": 0.09333333333333334,
      "grad_norm": 0.6443834325802673,
      "kl": 0.0002117156982421875,
      "learning_rate": 2e-06,
      "loss": 0.0879,
      "reward": 0.6600002646446228,
      "reward_std": 0.13175412267446518,
      "rewards/length_reward": 0.0656250137835741,
      "rewards/similarity_reward": 0.5943752527236938,
      "step": 35
    },
    {
      "completion_length": 265.6607360839844,
      "epoch": 0.096,
      "grad_norm": 0.7350551326118709,
      "kl": 0.000255584716796875,
      "learning_rate": 2e-06,
      "loss": 0.0763,
      "reward": 0.6379535496234894,
      "reward_std": 0.11777842044830322,
      "rewards/length_reward": 0.061160726472735405,
      "rewards/similarity_reward": 0.57679283618927,
      "step": 36
    },
    {
      "completion_length": 257.3928756713867,
      "epoch": 0.09866666666666667,
      "grad_norm": 0.7838614111079478,
      "kl": 0.0003414154052734375,
      "learning_rate": 2e-06,
      "loss": 0.0207,
      "reward": 0.6382779181003571,
      "reward_std": 0.10474509000778198,
      "rewards/length_reward": 0.07053572684526443,
      "rewards/similarity_reward": 0.567742183804512,
      "step": 37
    },
    {
      "completion_length": 281.02232360839844,
      "epoch": 0.10133333333333333,
      "grad_norm": 0.718502539431113,
      "kl": 0.000362396240234375,
      "learning_rate": 2e-06,
      "loss": 0.0634,
      "reward": 0.684599369764328,
      "reward_std": 0.12997651100158691,
      "rewards/length_reward": 0.0669642947614193,
      "rewards/similarity_reward": 0.6176350712776184,
      "step": 38
    },
    {
      "completion_length": 263.25447845458984,
      "epoch": 0.104,
      "grad_norm": 0.6560992159018658,
      "kl": 0.00020933151245117188,
      "learning_rate": 2e-06,
      "loss": 0.0628,
      "reward": 0.6398553550243378,
      "reward_std": 0.14855975657701492,
      "rewards/length_reward": 0.058035727590322495,
      "rewards/similarity_reward": 0.5818196535110474,
      "step": 39
    },
    {
      "completion_length": 315.5982360839844,
      "epoch": 0.10666666666666667,
      "grad_norm": 0.6477813051659901,
      "kl": 0.000324249267578125,
      "learning_rate": 2e-06,
      "loss": 0.1156,
      "reward": 0.57817542552948,
      "reward_std": 0.13820311054587364,
      "rewards/length_reward": 0.052678581327199936,
      "rewards/similarity_reward": 0.525496780872345,
      "step": 40
    },
    {
      "completion_length": 239.44644165039062,
      "epoch": 0.10933333333333334,
      "grad_norm": 0.6691411219790073,
      "kl": 0.0003085136413574219,
      "learning_rate": 2e-06,
      "loss": 0.0558,
      "reward": 0.6137453615665436,
      "reward_std": 0.13270848989486694,
      "rewards/length_reward": 0.061607152223587036,
      "rewards/similarity_reward": 0.5521382689476013,
      "step": 41
    },
    {
      "completion_length": 256.9866256713867,
      "epoch": 0.112,
      "grad_norm": 0.7883991158904521,
      "kl": 0.00034618377685546875,
      "learning_rate": 2e-06,
      "loss": 0.0302,
      "reward": 0.5703277885913849,
      "reward_std": 0.16237758100032806,
      "rewards/length_reward": 0.04910715110599995,
      "rewards/similarity_reward": 0.5212206840515137,
      "step": 42
    },
    {
      "completion_length": 294.6339416503906,
      "epoch": 0.11466666666666667,
      "grad_norm": 0.6538334220808693,
      "kl": 0.00020074844360351562,
      "learning_rate": 2e-06,
      "loss": 0.1178,
      "reward": 0.6531505286693573,
      "reward_std": 0.1386118158698082,
      "rewards/length_reward": 0.06964286789298058,
      "rewards/similarity_reward": 0.583507627248764,
      "step": 43
    },
    {
      "completion_length": 259.49554443359375,
      "epoch": 0.11733333333333333,
      "grad_norm": 0.6454158665655293,
      "kl": 0.0002779960632324219,
      "learning_rate": 2e-06,
      "loss": 0.0229,
      "reward": 0.6056394279003143,
      "reward_std": 0.1302252970635891,
      "rewards/length_reward": 0.07008930295705795,
      "rewards/similarity_reward": 0.5355501472949982,
      "step": 44
    },
    {
      "completion_length": 281.0982360839844,
      "epoch": 0.12,
      "grad_norm": 0.819280226455994,
      "kl": 0.00036144256591796875,
      "learning_rate": 2e-06,
      "loss": 0.1387,
      "reward": 0.5956102013587952,
      "reward_std": 0.1252099834382534,
      "rewards/length_reward": 0.05089286528527737,
      "rewards/similarity_reward": 0.5447173118591309,
      "step": 45
    },
    {
      "completion_length": 259.5848388671875,
      "epoch": 0.12266666666666666,
      "grad_norm": 0.7348999369116066,
      "kl": 0.00026702880859375,
      "learning_rate": 2e-06,
      "loss": 0.0863,
      "reward": 0.6511934101581573,
      "reward_std": 0.1133498027920723,
      "rewards/length_reward": 0.0625000111758709,
      "rewards/similarity_reward": 0.588693380355835,
      "step": 46
    },
    {
      "completion_length": 255.55358123779297,
      "epoch": 0.12533333333333332,
      "grad_norm": 0.8374758265958845,
      "kl": 0.00042057037353515625,
      "learning_rate": 2e-06,
      "loss": 0.094,
      "reward": 0.5409015715122223,
      "reward_std": 0.10493671149015427,
      "rewards/length_reward": 0.058928582817316055,
      "rewards/similarity_reward": 0.48197296261787415,
      "step": 47
    },
    {
      "completion_length": 262.93304443359375,
      "epoch": 0.128,
      "grad_norm": 0.6793546139834532,
      "kl": 0.000423431396484375,
      "learning_rate": 2e-06,
      "loss": 0.0649,
      "reward": 0.5910505056381226,
      "reward_std": 0.16320697963237762,
      "rewards/length_reward": 0.06428572721779346,
      "rewards/similarity_reward": 0.5267648100852966,
      "step": 48
    },
    {
      "completion_length": 274.53572845458984,
      "epoch": 0.13066666666666665,
      "grad_norm": 0.7145465386525555,
      "kl": 0.0003376007080078125,
      "learning_rate": 2e-06,
      "loss": 0.0209,
      "reward": 0.65211421251297,
      "reward_std": 0.16534826159477234,
      "rewards/length_reward": 0.06339287012815475,
      "rewards/similarity_reward": 0.588721364736557,
      "step": 49
    },
    {
      "completion_length": 267.5714340209961,
      "epoch": 0.13333333333333333,
      "grad_norm": 0.6146172464604014,
      "kl": 0.00035190582275390625,
      "learning_rate": 2e-06,
      "loss": 0.0452,
      "reward": 0.6328105330467224,
      "reward_std": 0.16772306710481644,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.5662927031517029,
      "step": 50
    },
    {
      "completion_length": 244.4910888671875,
      "epoch": 0.136,
      "grad_norm": 0.9373778001138064,
      "kl": 0.00045299530029296875,
      "learning_rate": 2e-06,
      "loss": 0.1341,
      "reward": 0.5600460767745972,
      "reward_std": 0.1334986537694931,
      "rewards/length_reward": 0.059375012293457985,
      "rewards/similarity_reward": 0.5006710439920425,
      "step": 51
    },
    {
      "completion_length": 251.4553680419922,
      "epoch": 0.13866666666666666,
      "grad_norm": 0.6828077611773323,
      "kl": 0.00024318695068359375,
      "learning_rate": 2e-06,
      "loss": 0.0511,
      "reward": 0.6347568333148956,
      "reward_std": 0.07518525794148445,
      "rewards/length_reward": 0.060267867520451546,
      "rewards/similarity_reward": 0.5744889676570892,
      "step": 52
    },
    {
      "completion_length": 258.74554443359375,
      "epoch": 0.14133333333333334,
      "grad_norm": 0.7291338070723046,
      "kl": 0.00031948089599609375,
      "learning_rate": 2e-06,
      "loss": -0.0178,
      "reward": 0.6101614236831665,
      "reward_std": 0.15437154471874237,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.5436435341835022,
      "step": 53
    },
    {
      "completion_length": 275.62501525878906,
      "epoch": 0.144,
      "grad_norm": 0.5749401912567113,
      "kl": 0.0002770423889160156,
      "learning_rate": 2e-06,
      "loss": 0.1447,
      "reward": 0.7116933763027191,
      "reward_std": 0.10556156933307648,
      "rewards/length_reward": 0.06964286789298058,
      "rewards/similarity_reward": 0.6420504748821259,
      "step": 54
    },
    {
      "completion_length": 297.88841247558594,
      "epoch": 0.14666666666666667,
      "grad_norm": 0.6712391841955945,
      "kl": 0.00043582916259765625,
      "learning_rate": 2e-06,
      "loss": 0.0543,
      "reward": 0.6665964424610138,
      "reward_std": 0.13994581252336502,
      "rewards/length_reward": 0.07232143357396126,
      "rewards/similarity_reward": 0.5942749679088593,
      "step": 55
    },
    {
      "completion_length": 261.2946548461914,
      "epoch": 0.14933333333333335,
      "grad_norm": 0.7543785996872888,
      "kl": 0.000606536865234375,
      "learning_rate": 2e-06,
      "loss": 0.0927,
      "reward": 0.5725409537553787,
      "reward_std": 0.19671519845724106,
      "rewards/length_reward": 0.05178572237491608,
      "rewards/similarity_reward": 0.5207552313804626,
      "step": 56
    },
    {
      "completion_length": 232.9196548461914,
      "epoch": 0.152,
      "grad_norm": 0.6310665650295014,
      "kl": 0.00034427642822265625,
      "learning_rate": 2e-06,
      "loss": 0.0251,
      "reward": 0.710765928030014,
      "reward_std": 0.11330895498394966,
      "rewards/length_reward": 0.07678571715950966,
      "rewards/similarity_reward": 0.6339801549911499,
      "step": 57
    },
    {
      "completion_length": 275.8839340209961,
      "epoch": 0.15466666666666667,
      "grad_norm": 0.6081646973355043,
      "kl": 0.00048542022705078125,
      "learning_rate": 2e-06,
      "loss": 0.0153,
      "reward": 0.634895533323288,
      "reward_std": 0.12484097108244896,
      "rewards/length_reward": 0.0736607201397419,
      "rewards/similarity_reward": 0.5612348020076752,
      "step": 58
    },
    {
      "completion_length": 296.9151916503906,
      "epoch": 0.15733333333333333,
      "grad_norm": 0.6547927211710071,
      "kl": 0.0003204345703125,
      "learning_rate": 2e-06,
      "loss": 0.101,
      "reward": 0.679833859205246,
      "reward_std": 0.13348159193992615,
      "rewards/length_reward": 0.07053572311997414,
      "rewards/similarity_reward": 0.6092981398105621,
      "step": 59
    },
    {
      "completion_length": 281.6919708251953,
      "epoch": 0.16,
      "grad_norm": 0.6398386646259059,
      "kl": 0.000492095947265625,
      "learning_rate": 2e-06,
      "loss": 0.0297,
      "reward": 0.6490921080112457,
      "reward_std": 0.14558688551187515,
      "rewards/length_reward": 0.07008930295705795,
      "rewards/similarity_reward": 0.5790028274059296,
      "step": 60
    },
    {
      "completion_length": 255.53126525878906,
      "epoch": 0.16266666666666665,
      "grad_norm": 0.7177382491967137,
      "kl": 0.0004730224609375,
      "learning_rate": 2e-06,
      "loss": 0.0431,
      "reward": 0.6999586224555969,
      "reward_std": 0.13094842806458473,
      "rewards/length_reward": 0.06383929774165154,
      "rewards/similarity_reward": 0.6361193954944611,
      "step": 61
    },
    {
      "completion_length": 269.15626525878906,
      "epoch": 0.16533333333333333,
      "grad_norm": 0.5891947471632187,
      "kl": 0.00038623809814453125,
      "learning_rate": 2e-06,
      "loss": 0.0092,
      "reward": 0.7129235565662384,
      "reward_std": 0.0941559188067913,
      "rewards/length_reward": 0.07455357909202576,
      "rewards/similarity_reward": 0.6383699178695679,
      "step": 62
    },
    {
      "completion_length": 294.62054443359375,
      "epoch": 0.168,
      "grad_norm": 0.8226717384917975,
      "kl": 0.0005388259887695312,
      "learning_rate": 2e-06,
      "loss": 0.0827,
      "reward": 0.6617253720760345,
      "reward_std": 0.10956033691763878,
      "rewards/length_reward": 0.06250001303851604,
      "rewards/similarity_reward": 0.5992253720760345,
      "step": 63
    },
    {
      "completion_length": 265.2410888671875,
      "epoch": 0.17066666666666666,
      "grad_norm": 0.7642480042855918,
      "kl": 0.0007228851318359375,
      "learning_rate": 2e-06,
      "loss": 0.0968,
      "reward": 0.6403190791606903,
      "reward_std": 0.1368250623345375,
      "rewards/length_reward": 0.06071429327130318,
      "rewards/similarity_reward": 0.5796047747135162,
      "step": 64
    },
    {
      "completion_length": 241.55358123779297,
      "epoch": 0.17333333333333334,
      "grad_norm": 0.8292179764665186,
      "kl": 0.00046443939208984375,
      "learning_rate": 2e-06,
      "loss": 0.065,
      "reward": 0.6242024898529053,
      "reward_std": 0.15151464939117432,
      "rewards/length_reward": 0.05491072125732899,
      "rewards/similarity_reward": 0.5692917704582214,
      "step": 65
    },
    {
      "completion_length": 234.7946548461914,
      "epoch": 0.176,
      "grad_norm": 0.6954246830187802,
      "kl": 0.00040721893310546875,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.6392101645469666,
      "reward_std": 0.13006018847227097,
      "rewards/length_reward": 0.06562500819563866,
      "rewards/similarity_reward": 0.5735851526260376,
      "step": 66
    },
    {
      "completion_length": 272.9776916503906,
      "epoch": 0.17866666666666667,
      "grad_norm": 0.8379515100728363,
      "kl": 0.0006618499755859375,
      "learning_rate": 2e-06,
      "loss": 0.0778,
      "reward": 0.6054639518260956,
      "reward_std": 0.15866044908761978,
      "rewards/length_reward": 0.056250011548399925,
      "rewards/similarity_reward": 0.5492139458656311,
      "step": 67
    },
    {
      "completion_length": 246.08482360839844,
      "epoch": 0.18133333333333335,
      "grad_norm": 0.6633622096307521,
      "kl": 0.000507354736328125,
      "learning_rate": 2e-06,
      "loss": 0.0327,
      "reward": 0.6771285533905029,
      "reward_std": 0.11619659885764122,
      "rewards/length_reward": 0.06696430034935474,
      "rewards/similarity_reward": 0.6101642847061157,
      "step": 68
    },
    {
      "completion_length": 260.4776916503906,
      "epoch": 0.184,
      "grad_norm": 0.7742612358878467,
      "kl": 0.0005159378051757812,
      "learning_rate": 2e-06,
      "loss": 0.0148,
      "reward": 0.711969256401062,
      "reward_std": 0.14254993572831154,
      "rewards/length_reward": 0.07276786491274834,
      "rewards/similarity_reward": 0.6392013728618622,
      "step": 69
    },
    {
      "completion_length": 218.6696548461914,
      "epoch": 0.18666666666666668,
      "grad_norm": 0.7878879808209311,
      "kl": 0.0006504058837890625,
      "learning_rate": 2e-06,
      "loss": 0.1186,
      "reward": 0.5868320167064667,
      "reward_std": 0.139366053044796,
      "rewards/length_reward": 0.050892867147922516,
      "rewards/similarity_reward": 0.5359391123056412,
      "step": 70
    },
    {
      "completion_length": 273.07591247558594,
      "epoch": 0.18933333333333333,
      "grad_norm": 0.6879154147040387,
      "kl": 0.00072479248046875,
      "learning_rate": 2e-06,
      "loss": 0.0361,
      "reward": 0.5751383006572723,
      "reward_std": 0.16684868186712265,
      "rewards/length_reward": 0.061607155948877335,
      "rewards/similarity_reward": 0.5135311335325241,
      "step": 71
    },
    {
      "completion_length": 271.7053680419922,
      "epoch": 0.192,
      "grad_norm": 0.6918784121385549,
      "kl": 0.0006055831909179688,
      "learning_rate": 2e-06,
      "loss": 0.0412,
      "reward": 0.6015307903289795,
      "reward_std": 0.11276621744036674,
      "rewards/length_reward": 0.06383929401636124,
      "rewards/similarity_reward": 0.5376915037631989,
      "step": 72
    },
    {
      "completion_length": 239.18304443359375,
      "epoch": 0.19466666666666665,
      "grad_norm": 0.6763207999389166,
      "kl": 0.00046539306640625,
      "learning_rate": 2e-06,
      "loss": 0.0378,
      "reward": 0.738762378692627,
      "reward_std": 0.07676012441515923,
      "rewards/length_reward": 0.0669642947614193,
      "rewards/similarity_reward": 0.671798050403595,
      "step": 73
    },
    {
      "completion_length": 298.2321472167969,
      "epoch": 0.19733333333333333,
      "grad_norm": 0.7183360474036766,
      "kl": 0.00069427490234375,
      "learning_rate": 2e-06,
      "loss": 0.1536,
      "reward": 0.6193753182888031,
      "reward_std": 0.13081318512558937,
      "rewards/length_reward": 0.055803585797548294,
      "rewards/similarity_reward": 0.5635717213153839,
      "step": 74
    },
    {
      "completion_length": 278.71876525878906,
      "epoch": 0.2,
      "grad_norm": 0.6422209743450289,
      "kl": 0.0005578994750976562,
      "learning_rate": 2e-06,
      "loss": 0.1141,
      "reward": 0.7105484306812286,
      "reward_std": 0.1271527223289013,
      "rewards/length_reward": 0.0758928656578064,
      "rewards/similarity_reward": 0.6346555352210999,
      "step": 75
    },
    {
      "completion_length": 232.0759048461914,
      "epoch": 0.20266666666666666,
      "grad_norm": 0.8972600571762434,
      "kl": 0.0006427764892578125,
      "learning_rate": 2e-06,
      "loss": 0.051,
      "reward": 0.5786014497280121,
      "reward_std": 0.13573814183473587,
      "rewards/length_reward": 0.06964286416769028,
      "rewards/similarity_reward": 0.5089586079120636,
      "step": 76
    },
    {
      "completion_length": 281.2232208251953,
      "epoch": 0.20533333333333334,
      "grad_norm": 0.8130156685147141,
      "kl": 0.00061798095703125,
      "learning_rate": 2e-06,
      "loss": 0.1302,
      "reward": 0.6210056841373444,
      "reward_std": 0.19511007517576218,
      "rewards/length_reward": 0.053125010803341866,
      "rewards/similarity_reward": 0.5678807199001312,
      "step": 77
    },
    {
      "completion_length": 258.6651916503906,
      "epoch": 0.208,
      "grad_norm": 0.7255155232620163,
      "kl": 0.0008106231689453125,
      "learning_rate": 2e-06,
      "loss": 0.0153,
      "reward": 0.619497686624527,
      "reward_std": 0.10088447853922844,
      "rewards/length_reward": 0.060267867520451546,
      "rewards/similarity_reward": 0.5592298209667206,
      "step": 78
    },
    {
      "completion_length": 255.79466247558594,
      "epoch": 0.21066666666666667,
      "grad_norm": 0.6729672044279214,
      "kl": 0.000576019287109375,
      "learning_rate": 2e-06,
      "loss": 0.0819,
      "reward": 0.7009203135967255,
      "reward_std": 0.10652491822838783,
      "rewards/length_reward": 0.07142858020961285,
      "rewards/similarity_reward": 0.6294918060302734,
      "step": 79
    },
    {
      "completion_length": 275.15626525878906,
      "epoch": 0.21333333333333335,
      "grad_norm": 0.7063326351608399,
      "kl": 0.0008029937744140625,
      "learning_rate": 2e-06,
      "loss": 0.106,
      "reward": 0.6244928240776062,
      "reward_std": 0.14966875314712524,
      "rewards/length_reward": 0.056696439161896706,
      "rewards/similarity_reward": 0.5677963495254517,
      "step": 80
    },
    {
      "completion_length": 280.9464416503906,
      "epoch": 0.216,
      "grad_norm": 0.5928309311542868,
      "kl": 0.00060272216796875,
      "learning_rate": 2e-06,
      "loss": 0.0107,
      "reward": 0.6575084030628204,
      "reward_std": 0.10019119456410408,
      "rewards/length_reward": 0.06964286789298058,
      "rewards/similarity_reward": 0.587865561246872,
      "step": 81
    },
    {
      "completion_length": 232.42858123779297,
      "epoch": 0.21866666666666668,
      "grad_norm": 0.728258017728098,
      "kl": 0.0008029937744140625,
      "learning_rate": 2e-06,
      "loss": 0.0327,
      "reward": 0.6456558704376221,
      "reward_std": 0.10026798397302628,
      "rewards/length_reward": 0.07187500596046448,
      "rewards/similarity_reward": 0.5737808644771576,
      "step": 82
    },
    {
      "completion_length": 272.4241180419922,
      "epoch": 0.22133333333333333,
      "grad_norm": 0.7474863840166143,
      "kl": 0.0006437301635742188,
      "learning_rate": 2e-06,
      "loss": 0.0835,
      "reward": 0.6149618327617645,
      "reward_std": 0.11428236961364746,
      "rewards/length_reward": 0.06205357797443867,
      "rewards/similarity_reward": 0.552908256649971,
      "step": 83
    },
    {
      "completion_length": 281.40179443359375,
      "epoch": 0.224,
      "grad_norm": 0.5698111928088367,
      "kl": 0.0008392333984375,
      "learning_rate": 2e-06,
      "loss": 0.0067,
      "reward": 0.6963447332382202,
      "reward_std": 0.10215538740158081,
      "rewards/length_reward": 0.06830358132719994,
      "rewards/similarity_reward": 0.6280410885810852,
      "step": 84
    },
    {
      "completion_length": 249.6696548461914,
      "epoch": 0.22666666666666666,
      "grad_norm": 0.774725228553314,
      "kl": 0.0007877349853515625,
      "learning_rate": 2e-06,
      "loss": 0.0906,
      "reward": 0.6118488311767578,
      "reward_std": 0.14368778094649315,
      "rewards/length_reward": 0.06428572349250317,
      "rewards/similarity_reward": 0.5475630760192871,
      "step": 85
    },
    {
      "completion_length": 265.3482208251953,
      "epoch": 0.22933333333333333,
      "grad_norm": 0.8923548112445079,
      "kl": 0.0011768341064453125,
      "learning_rate": 2e-06,
      "loss": 0.0958,
      "reward": 0.6147017180919647,
      "reward_std": 0.1361084319651127,
      "rewards/length_reward": 0.057589296251535416,
      "rewards/similarity_reward": 0.557112455368042,
      "step": 86
    },
    {
      "completion_length": 251.7901840209961,
      "epoch": 0.232,
      "grad_norm": 0.618718290820211,
      "kl": 0.0007457733154296875,
      "learning_rate": 2e-06,
      "loss": 0.0706,
      "reward": 0.6785232722759247,
      "reward_std": 0.09245472028851509,
      "rewards/length_reward": 0.07500000670552254,
      "rewards/similarity_reward": 0.6035232245922089,
      "step": 87
    },
    {
      "completion_length": 311.21876525878906,
      "epoch": 0.23466666666666666,
      "grad_norm": 0.5515407468114694,
      "kl": 0.000797271728515625,
      "learning_rate": 2e-06,
      "loss": 0.012,
      "reward": 0.6955881416797638,
      "reward_std": 0.10647468641400337,
      "rewards/length_reward": 0.08303571492433548,
      "rewards/similarity_reward": 0.6125523746013641,
      "step": 88
    },
    {
      "completion_length": 241.92858123779297,
      "epoch": 0.23733333333333334,
      "grad_norm": 0.7246692380980303,
      "kl": 0.0008525848388671875,
      "learning_rate": 2e-06,
      "loss": 0.0111,
      "reward": 0.7196685373783112,
      "reward_std": 0.11786854639649391,
      "rewards/length_reward": 0.06562501564621925,
      "rewards/similarity_reward": 0.6540435552597046,
      "step": 89
    },
    {
      "completion_length": 266.3482208251953,
      "epoch": 0.24,
      "grad_norm": 0.8337410916485611,
      "kl": 0.00109100341796875,
      "learning_rate": 2e-06,
      "loss": 0.0753,
      "reward": 0.6227089762687683,
      "reward_std": 0.11949346587061882,
      "rewards/length_reward": 0.05178572237491608,
      "rewards/similarity_reward": 0.570923238992691,
      "step": 90
    },
    {
      "completion_length": 312.4553680419922,
      "epoch": 0.24266666666666667,
      "grad_norm": 0.7247658170183164,
      "kl": 0.0009250640869140625,
      "learning_rate": 2e-06,
      "loss": 0.074,
      "reward": 0.6411847472190857,
      "reward_std": 0.15169727057218552,
      "rewards/length_reward": 0.06651787087321281,
      "rewards/similarity_reward": 0.5746668875217438,
      "step": 91
    },
    {
      "completion_length": 240.4821548461914,
      "epoch": 0.24533333333333332,
      "grad_norm": 0.7784189357920823,
      "kl": 0.000835418701171875,
      "learning_rate": 2e-06,
      "loss": 0.0335,
      "reward": 0.7014745771884918,
      "reward_std": 0.08627640455961227,
      "rewards/length_reward": 0.06517857871949673,
      "rewards/similarity_reward": 0.6362960338592529,
      "step": 92
    },
    {
      "completion_length": 253.05805206298828,
      "epoch": 0.248,
      "grad_norm": 0.7201282634879667,
      "kl": 0.000904083251953125,
      "learning_rate": 2e-06,
      "loss": 0.0981,
      "reward": 0.6915363669395447,
      "reward_std": 0.14938431978225708,
      "rewards/length_reward": 0.06875001266598701,
      "rewards/similarity_reward": 0.6227863430976868,
      "step": 93
    },
    {
      "completion_length": 281.4776916503906,
      "epoch": 0.25066666666666665,
      "grad_norm": 0.6973337009868545,
      "kl": 0.0007686614990234375,
      "learning_rate": 2e-06,
      "loss": 0.0789,
      "reward": 0.6481258869171143,
      "reward_std": 0.10927876830101013,
      "rewards/length_reward": 0.06830358132719994,
      "rewards/similarity_reward": 0.5798223316669464,
      "step": 94
    },
    {
      "completion_length": 205.28125762939453,
      "epoch": 0.25333333333333335,
      "grad_norm": 0.8693659568274648,
      "kl": 0.00103759765625,
      "learning_rate": 2e-06,
      "loss": 0.0431,
      "reward": 0.6439125239849091,
      "reward_std": 0.11249563843011856,
      "rewards/length_reward": 0.058035727590322495,
      "rewards/similarity_reward": 0.5858767628669739,
      "step": 95
    },
    {
      "completion_length": 292.4241180419922,
      "epoch": 0.256,
      "grad_norm": 0.5457303629614552,
      "kl": 0.0009708404541015625,
      "learning_rate": 2e-06,
      "loss": 0.0148,
      "reward": 0.7119653820991516,
      "reward_std": 0.1094476543366909,
      "rewards/length_reward": 0.08169642835855484,
      "rewards/similarity_reward": 0.630268931388855,
      "step": 96
    },
    {
      "completion_length": 300.5089416503906,
      "epoch": 0.25866666666666666,
      "grad_norm": 0.5831645575431905,
      "kl": 0.0009822845458984375,
      "learning_rate": 2e-06,
      "loss": 0.0146,
      "reward": 0.6134711802005768,
      "reward_std": 0.10632903501391411,
      "rewards/length_reward": 0.07142858020961285,
      "rewards/similarity_reward": 0.5420425981283188,
      "step": 97
    },
    {
      "completion_length": 239.30358123779297,
      "epoch": 0.2613333333333333,
      "grad_norm": 0.7748344420880229,
      "kl": 0.0009174346923828125,
      "learning_rate": 2e-06,
      "loss": 0.0451,
      "reward": 0.682037353515625,
      "reward_std": 0.08752915635704994,
      "rewards/length_reward": 0.07410715520381927,
      "rewards/similarity_reward": 0.6079301834106445,
      "step": 98
    },
    {
      "completion_length": 247.00001525878906,
      "epoch": 0.264,
      "grad_norm": 0.6574205211811521,
      "kl": 0.00130462646484375,
      "learning_rate": 2e-06,
      "loss": 0.0383,
      "reward": 0.6871787905693054,
      "reward_std": 0.12282633036375046,
      "rewards/length_reward": 0.06607143767178059,
      "rewards/similarity_reward": 0.6211073100566864,
      "step": 99
    },
    {
      "completion_length": 255.9866180419922,
      "epoch": 0.26666666666666666,
      "grad_norm": 0.6550349068969015,
      "kl": 0.0011653900146484375,
      "learning_rate": 2e-06,
      "loss": 0.031,
      "reward": 0.6847736537456512,
      "reward_std": 0.11585507914423943,
      "rewards/length_reward": 0.07767857611179352,
      "rewards/similarity_reward": 0.6070950329303741,
      "step": 100
    },
    {
      "completion_length": 302.05804443359375,
      "epoch": 0.2693333333333333,
      "grad_norm": 0.6581037881537074,
      "kl": 0.0008869171142578125,
      "learning_rate": 2e-06,
      "loss": 0.0275,
      "reward": 0.7058568000793457,
      "reward_std": 0.1038212925195694,
      "rewards/length_reward": 0.0736607238650322,
      "rewards/similarity_reward": 0.6321960687637329,
      "step": 101
    },
    {
      "completion_length": 252.37055206298828,
      "epoch": 0.272,
      "grad_norm": 0.7659892153308909,
      "kl": 0.0009002685546875,
      "learning_rate": 2e-06,
      "loss": 0.0099,
      "reward": 0.6600384712219238,
      "reward_std": 0.11620288342237473,
      "rewards/length_reward": 0.06964286416769028,
      "rewards/similarity_reward": 0.5903956294059753,
      "step": 102
    },
    {
      "completion_length": 260.9910888671875,
      "epoch": 0.27466666666666667,
      "grad_norm": 0.7873803837848522,
      "kl": 0.0011749267578125,
      "learning_rate": 2e-06,
      "loss": 0.067,
      "reward": 0.647940456867218,
      "reward_std": 0.12016744539141655,
      "rewards/length_reward": 0.060267869383096695,
      "rewards/similarity_reward": 0.587672621011734,
      "step": 103
    },
    {
      "completion_length": 303.0982208251953,
      "epoch": 0.2773333333333333,
      "grad_norm": 0.6698404225945914,
      "kl": 0.0008029937744140625,
      "learning_rate": 2e-06,
      "loss": 0.0326,
      "reward": 0.6724097728729248,
      "reward_std": 0.08444990962743759,
      "rewards/length_reward": 0.07232143729925156,
      "rewards/similarity_reward": 0.600088357925415,
      "step": 104
    },
    {
      "completion_length": 263.43751525878906,
      "epoch": 0.28,
      "grad_norm": 0.7035133703979454,
      "kl": 0.0010471343994140625,
      "learning_rate": 2e-06,
      "loss": 0.0377,
      "reward": 0.6536130607128143,
      "reward_std": 0.16632840782403946,
      "rewards/length_reward": 0.06741072982549667,
      "rewards/similarity_reward": 0.5862023532390594,
      "step": 105
    },
    {
      "completion_length": 265.3526916503906,
      "epoch": 0.2826666666666667,
      "grad_norm": 0.7008056428586112,
      "kl": 0.0012378692626953125,
      "learning_rate": 2e-06,
      "loss": 0.0104,
      "reward": 0.6467622220516205,
      "reward_std": 0.11927535384893417,
      "rewards/length_reward": 0.06651787087321281,
      "rewards/similarity_reward": 0.5802443325519562,
      "step": 106
    },
    {
      "completion_length": 275.6026916503906,
      "epoch": 0.2853333333333333,
      "grad_norm": 0.6158480236246975,
      "kl": 0.001171112060546875,
      "learning_rate": 2e-06,
      "loss": 0.0208,
      "reward": 0.6745641827583313,
      "reward_std": 0.11500725150108337,
      "rewards/length_reward": 0.06964286416769028,
      "rewards/similarity_reward": 0.604921281337738,
      "step": 107
    },
    {
      "completion_length": 260.33036041259766,
      "epoch": 0.288,
      "grad_norm": 0.7224564328760787,
      "kl": 0.000965118408203125,
      "learning_rate": 2e-06,
      "loss": 0.0761,
      "reward": 0.5630394518375397,
      "reward_std": 0.1106732226908207,
      "rewards/length_reward": 0.053571442142128944,
      "rewards/similarity_reward": 0.5094679594039917,
      "step": 108
    },
    {
      "completion_length": 243.68751525878906,
      "epoch": 0.2906666666666667,
      "grad_norm": 0.8254406902750699,
      "kl": 0.001445770263671875,
      "learning_rate": 2e-06,
      "loss": 0.0425,
      "reward": 0.6382045447826385,
      "reward_std": 0.12056771293282509,
      "rewards/length_reward": 0.062053581699728966,
      "rewards/similarity_reward": 0.5761510133743286,
      "step": 109
    },
    {
      "completion_length": 224.87501525878906,
      "epoch": 0.29333333333333333,
      "grad_norm": 0.6268606551971334,
      "kl": 0.0011119842529296875,
      "learning_rate": 2e-06,
      "loss": 0.0105,
      "reward": 0.5971523225307465,
      "reward_std": 0.10623245313763618,
      "rewards/length_reward": 0.06517858244478703,
      "rewards/similarity_reward": 0.5319737493991852,
      "step": 110
    },
    {
      "completion_length": 259.1339340209961,
      "epoch": 0.296,
      "grad_norm": 0.6687342397274337,
      "kl": 0.00101470947265625,
      "learning_rate": 2e-06,
      "loss": 0.1205,
      "reward": 0.6370533108711243,
      "reward_std": 0.12068188562989235,
      "rewards/length_reward": 0.06294644251465797,
      "rewards/similarity_reward": 0.5741068720817566,
      "step": 111
    },
    {
      "completion_length": 249.50000762939453,
      "epoch": 0.2986666666666667,
      "grad_norm": 0.6806425164227126,
      "kl": 0.001422882080078125,
      "learning_rate": 2e-06,
      "loss": 0.052,
      "reward": 0.6264582276344299,
      "reward_std": 0.11265718936920166,
      "rewards/length_reward": 0.06428572908043861,
      "rewards/similarity_reward": 0.562172532081604,
      "step": 112
    },
    {
      "completion_length": 270.3883972167969,
      "epoch": 0.30133333333333334,
      "grad_norm": 0.7014564695453412,
      "kl": 0.0009479522705078125,
      "learning_rate": 2e-06,
      "loss": 0.1126,
      "reward": 0.7100532650947571,
      "reward_std": 0.10190926492214203,
      "rewards/length_reward": 0.07857143878936768,
      "rewards/similarity_reward": 0.6314818263053894,
      "step": 113
    },
    {
      "completion_length": 271.99554443359375,
      "epoch": 0.304,
      "grad_norm": 0.8072459229444039,
      "kl": 0.001361846923828125,
      "learning_rate": 2e-06,
      "loss": 0.1128,
      "reward": 0.6362347304821014,
      "reward_std": 0.13582201302051544,
      "rewards/length_reward": 0.07008929550647736,
      "rewards/similarity_reward": 0.5661454498767853,
      "step": 114
    },
    {
      "completion_length": 223.00447845458984,
      "epoch": 0.30666666666666664,
      "grad_norm": 0.8550809137589263,
      "kl": 0.001277923583984375,
      "learning_rate": 2e-06,
      "loss": 0.1073,
      "reward": 0.6763125956058502,
      "reward_std": 0.11202903836965561,
      "rewards/length_reward": 0.06607143580913544,
      "rewards/similarity_reward": 0.6102411448955536,
      "step": 115
    },
    {
      "completion_length": 279.54466247558594,
      "epoch": 0.30933333333333335,
      "grad_norm": 0.7634314721408118,
      "kl": 0.001033782958984375,
      "learning_rate": 2e-06,
      "loss": 0.1038,
      "reward": 0.6390451192855835,
      "reward_std": 0.11136174201965332,
      "rewards/length_reward": 0.0714285746216774,
      "rewards/similarity_reward": 0.5676165223121643,
      "step": 116
    },
    {
      "completion_length": 251.08929443359375,
      "epoch": 0.312,
      "grad_norm": 0.8512986272863483,
      "kl": 0.0017547607421875,
      "learning_rate": 2e-06,
      "loss": 0.027,
      "reward": 0.5662170350551605,
      "reward_std": 0.1223873421549797,
      "rewards/length_reward": 0.051785726100206375,
      "rewards/similarity_reward": 0.5144313126802444,
      "step": 117
    },
    {
      "completion_length": 279.3526916503906,
      "epoch": 0.31466666666666665,
      "grad_norm": 0.5648142162422048,
      "kl": 0.001270294189453125,
      "learning_rate": 2e-06,
      "loss": 0.058,
      "reward": 0.6961429119110107,
      "reward_std": 0.08081653341650963,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.6296250522136688,
      "step": 118
    },
    {
      "completion_length": 228.95536041259766,
      "epoch": 0.31733333333333336,
      "grad_norm": 0.87246537686519,
      "kl": 0.00152587890625,
      "learning_rate": 2e-06,
      "loss": 0.0655,
      "reward": 0.5725010931491852,
      "reward_std": 0.13679825142025948,
      "rewards/length_reward": 0.051785724237561226,
      "rewards/similarity_reward": 0.5207154005765915,
      "step": 119
    },
    {
      "completion_length": 213.12500762939453,
      "epoch": 0.32,
      "grad_norm": 0.90775103522726,
      "kl": 0.00186920166015625,
      "learning_rate": 2e-06,
      "loss": 0.0752,
      "reward": 0.5904355347156525,
      "reward_std": 0.13604873418807983,
      "rewards/length_reward": 0.060714298859238625,
      "rewards/similarity_reward": 0.5297212451696396,
      "step": 120
    },
    {
      "completion_length": 272.4598388671875,
      "epoch": 0.32266666666666666,
      "grad_norm": 0.679207876261083,
      "kl": 0.001651763916015625,
      "learning_rate": 2e-06,
      "loss": 0.0208,
      "reward": 0.5465417206287384,
      "reward_std": 0.11999019607901573,
      "rewards/length_reward": 0.060714296996593475,
      "rewards/similarity_reward": 0.48582740128040314,
      "step": 121
    },
    {
      "completion_length": 264.77679443359375,
      "epoch": 0.3253333333333333,
      "grad_norm": 0.6586415817875254,
      "kl": 0.0012359619140625,
      "learning_rate": 2e-06,
      "loss": 0.0806,
      "reward": 0.68158820271492,
      "reward_std": 0.1025921143591404,
      "rewards/length_reward": 0.07008929923176765,
      "rewards/similarity_reward": 0.6114989817142487,
      "step": 122
    },
    {
      "completion_length": 256.1607208251953,
      "epoch": 0.328,
      "grad_norm": 0.7273109756128965,
      "kl": 0.00136566162109375,
      "learning_rate": 2e-06,
      "loss": 0.0542,
      "reward": 0.6582387089729309,
      "reward_std": 0.10396385937929153,
      "rewards/length_reward": 0.062053581699728966,
      "rewards/similarity_reward": 0.5961851477622986,
      "step": 123
    },
    {
      "completion_length": 258.7723388671875,
      "epoch": 0.33066666666666666,
      "grad_norm": 0.7158563788139325,
      "kl": 0.0016632080078125,
      "learning_rate": 2e-06,
      "loss": 0.0495,
      "reward": 0.680817037820816,
      "reward_std": 0.15591518580913544,
      "rewards/length_reward": 0.0669643022119999,
      "rewards/similarity_reward": 0.6138526499271393,
      "step": 124
    },
    {
      "completion_length": 287.1026916503906,
      "epoch": 0.3333333333333333,
      "grad_norm": 0.7257840755435857,
      "kl": 0.0021820068359375,
      "learning_rate": 2e-06,
      "loss": 0.1379,
      "reward": 0.6818304061889648,
      "reward_std": 0.14189621806144714,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.6153125464916229,
      "step": 125
    },
    {
      "completion_length": 272.3794860839844,
      "epoch": 0.336,
      "grad_norm": 0.716044862480554,
      "kl": 0.00128936767578125,
      "learning_rate": 2e-06,
      "loss": 0.0606,
      "reward": 0.607832282781601,
      "reward_std": 0.10822133347392082,
      "rewards/length_reward": 0.06517858244478703,
      "rewards/similarity_reward": 0.5426536649465561,
      "step": 126
    },
    {
      "completion_length": 267.2008972167969,
      "epoch": 0.33866666666666667,
      "grad_norm": 0.7078650786713803,
      "kl": 0.0017242431640625,
      "learning_rate": 2e-06,
      "loss": 0.0553,
      "reward": 0.6737089157104492,
      "reward_std": 0.12828165292739868,
      "rewards/length_reward": 0.06830358132719994,
      "rewards/similarity_reward": 0.6054053008556366,
      "step": 127
    },
    {
      "completion_length": 247.7276840209961,
      "epoch": 0.3413333333333333,
      "grad_norm": 0.9316222904188678,
      "kl": 0.0016021728515625,
      "learning_rate": 2e-06,
      "loss": 0.0704,
      "reward": 0.6324192881584167,
      "reward_std": 0.09700313210487366,
      "rewards/length_reward": 0.056250011548399925,
      "rewards/similarity_reward": 0.5761693120002747,
      "step": 128
    },
    {
      "completion_length": 266.2946548461914,
      "epoch": 0.344,
      "grad_norm": 0.6821391109445131,
      "kl": 0.00136566162109375,
      "learning_rate": 2e-06,
      "loss": 0.0922,
      "reward": 0.600318193435669,
      "reward_std": 0.14177118614315987,
      "rewards/length_reward": 0.05892857722938061,
      "rewards/similarity_reward": 0.5413895845413208,
      "step": 129
    },
    {
      "completion_length": 260.25447845458984,
      "epoch": 0.3466666666666667,
      "grad_norm": 0.6135827945376331,
      "kl": 0.001003265380859375,
      "learning_rate": 2e-06,
      "loss": 0.0975,
      "reward": 0.6941237151622772,
      "reward_std": 0.11082329601049423,
      "rewards/length_reward": 0.07232144102454185,
      "rewards/similarity_reward": 0.6218023300170898,
      "step": 130
    },
    {
      "completion_length": 301.4598388671875,
      "epoch": 0.34933333333333333,
      "grad_norm": 0.727973261017594,
      "kl": 0.00146484375,
      "learning_rate": 2e-06,
      "loss": 0.1049,
      "reward": 0.6211672425270081,
      "reward_std": 0.1372002437710762,
      "rewards/length_reward": 0.05625000782310963,
      "rewards/similarity_reward": 0.5649172067642212,
      "step": 131
    },
    {
      "completion_length": 235.4910888671875,
      "epoch": 0.352,
      "grad_norm": 0.7386524195731257,
      "kl": 0.0012664794921875,
      "learning_rate": 2e-06,
      "loss": 0.0175,
      "reward": 0.6908943951129913,
      "reward_std": 0.09667576849460602,
      "rewards/length_reward": 0.06830358132719994,
      "rewards/similarity_reward": 0.6225908398628235,
      "step": 132
    },
    {
      "completion_length": 220.09376525878906,
      "epoch": 0.3546666666666667,
      "grad_norm": 0.6544388171069107,
      "kl": 0.00138092041015625,
      "learning_rate": 2e-06,
      "loss": 0.0025,
      "reward": 0.6801804006099701,
      "reward_std": 0.11540298722684383,
      "rewards/length_reward": 0.06830358132719994,
      "rewards/similarity_reward": 0.6118768155574799,
      "step": 133
    },
    {
      "completion_length": 273.5714340209961,
      "epoch": 0.35733333333333334,
      "grad_norm": 0.5808123143329609,
      "kl": 0.0009899139404296875,
      "learning_rate": 2e-06,
      "loss": 0.0302,
      "reward": 0.6981923878192902,
      "reward_std": 0.12756088376045227,
      "rewards/length_reward": 0.071428582072258,
      "rewards/similarity_reward": 0.6267638206481934,
      "step": 134
    },
    {
      "completion_length": 273.83483123779297,
      "epoch": 0.36,
      "grad_norm": 0.6257800546867194,
      "kl": 0.001750946044921875,
      "learning_rate": 2e-06,
      "loss": 0.0255,
      "reward": 0.6324395537376404,
      "reward_std": 0.11662106215953827,
      "rewards/length_reward": 0.07276786491274834,
      "rewards/similarity_reward": 0.5596716403961182,
      "step": 135
    },
    {
      "completion_length": 209.4821548461914,
      "epoch": 0.3626666666666667,
      "grad_norm": 0.7210088344519984,
      "kl": 0.00167083740234375,
      "learning_rate": 2e-06,
      "loss": 0.0129,
      "reward": 0.6599233448505402,
      "reward_std": 0.08075730502605438,
      "rewards/length_reward": 0.06339286640286446,
      "rewards/similarity_reward": 0.5965304374694824,
      "step": 136
    },
    {
      "completion_length": 258.7232208251953,
      "epoch": 0.36533333333333334,
      "grad_norm": 0.7922066598101163,
      "kl": 0.001499176025390625,
      "learning_rate": 2e-06,
      "loss": 0.0838,
      "reward": 0.6245545744895935,
      "reward_std": 0.12391168251633644,
      "rewards/length_reward": 0.0625000111758709,
      "rewards/similarity_reward": 0.5620545446872711,
      "step": 137
    },
    {
      "completion_length": 290.34822845458984,
      "epoch": 0.368,
      "grad_norm": 0.6230240946470257,
      "kl": 0.00208282470703125,
      "learning_rate": 2e-06,
      "loss": 0.0447,
      "reward": 0.6634125411510468,
      "reward_std": 0.10979421064257622,
      "rewards/length_reward": 0.06428572535514832,
      "rewards/similarity_reward": 0.599126785993576,
      "step": 138
    },
    {
      "completion_length": 265.3839416503906,
      "epoch": 0.37066666666666664,
      "grad_norm": 0.7078887986803306,
      "kl": 0.002201080322265625,
      "learning_rate": 2e-06,
      "loss": 0.0458,
      "reward": 0.70221146941185,
      "reward_std": 0.10299094393849373,
      "rewards/length_reward": 0.06964286789298058,
      "rewards/similarity_reward": 0.6325685381889343,
      "step": 139
    },
    {
      "completion_length": 265.7678680419922,
      "epoch": 0.37333333333333335,
      "grad_norm": 0.7842037190011205,
      "kl": 0.001491546630859375,
      "learning_rate": 2e-06,
      "loss": 0.0973,
      "reward": 0.644274890422821,
      "reward_std": 0.10034830123186111,
      "rewards/length_reward": 0.061160728335380554,
      "rewards/similarity_reward": 0.5831141769886017,
      "step": 140
    },
    {
      "completion_length": 249.66964721679688,
      "epoch": 0.376,
      "grad_norm": 0.567490680527846,
      "kl": 0.001689910888671875,
      "learning_rate": 2e-06,
      "loss": -0.024,
      "reward": 0.7013588547706604,
      "reward_std": 0.0840650200843811,
      "rewards/length_reward": 0.06651786342263222,
      "rewards/similarity_reward": 0.6348409652709961,
      "step": 141
    },
    {
      "completion_length": 226.25447845458984,
      "epoch": 0.37866666666666665,
      "grad_norm": 0.912319551814626,
      "kl": 0.001934051513671875,
      "learning_rate": 2e-06,
      "loss": 0.0961,
      "reward": 0.6543315052986145,
      "reward_std": 0.10859474539756775,
      "rewards/length_reward": 0.06607143953442574,
      "rewards/similarity_reward": 0.5882599949836731,
      "step": 142
    },
    {
      "completion_length": 298.8616180419922,
      "epoch": 0.38133333333333336,
      "grad_norm": 0.5902099764974178,
      "kl": 0.001720428466796875,
      "learning_rate": 2e-06,
      "loss": -0.0059,
      "reward": 0.7150795757770538,
      "reward_std": 0.08074402436614037,
      "rewards/length_reward": 0.0781250111758709,
      "rewards/similarity_reward": 0.6369545459747314,
      "step": 143
    },
    {
      "completion_length": 258.6026916503906,
      "epoch": 0.384,
      "grad_norm": 0.7182388748669096,
      "kl": 0.0016632080078125,
      "learning_rate": 2e-06,
      "loss": 0.073,
      "reward": 0.6721713840961456,
      "reward_std": 0.10838266462087631,
      "rewards/length_reward": 0.07321429625153542,
      "rewards/similarity_reward": 0.5989570617675781,
      "step": 144
    },
    {
      "completion_length": 249.93304443359375,
      "epoch": 0.38666666666666666,
      "grad_norm": 0.736716904979314,
      "kl": 0.001873016357421875,
      "learning_rate": 2e-06,
      "loss": 0.0098,
      "reward": 0.6520076394081116,
      "reward_std": 0.08915667980909348,
      "rewards/length_reward": 0.059821439906954765,
      "rewards/similarity_reward": 0.5921862125396729,
      "step": 145
    },
    {
      "completion_length": 249.5401840209961,
      "epoch": 0.3893333333333333,
      "grad_norm": 0.696504045867252,
      "kl": 0.0015106201171875,
      "learning_rate": 2e-06,
      "loss": 0.0287,
      "reward": 0.6746790409088135,
      "reward_std": 0.10656709223985672,
      "rewards/length_reward": 0.06785715371370316,
      "rewards/similarity_reward": 0.6068219244480133,
      "step": 146
    },
    {
      "completion_length": 335.7098388671875,
      "epoch": 0.392,
      "grad_norm": 0.5956601576776329,
      "kl": 0.001354217529296875,
      "learning_rate": 2e-06,
      "loss": 0.0678,
      "reward": 0.6324039399623871,
      "reward_std": 0.0887768529355526,
      "rewards/length_reward": 0.07053572684526443,
      "rewards/similarity_reward": 0.5618681907653809,
      "step": 147
    },
    {
      "completion_length": 245.93304443359375,
      "epoch": 0.39466666666666667,
      "grad_norm": 0.7100692950335996,
      "kl": 0.001987457275390625,
      "learning_rate": 2e-06,
      "loss": 0.0427,
      "reward": 0.6061528325080872,
      "reward_std": 0.09401173889636993,
      "rewards/length_reward": 0.06830358132719994,
      "rewards/similarity_reward": 0.5378492623567581,
      "step": 148
    },
    {
      "completion_length": 220.54911041259766,
      "epoch": 0.3973333333333333,
      "grad_norm": 0.8672496323443106,
      "kl": 0.00212860107421875,
      "learning_rate": 2e-06,
      "loss": 0.0339,
      "reward": 0.6478115916252136,
      "reward_std": 0.12049812451004982,
      "rewards/length_reward": 0.06562500819563866,
      "rewards/similarity_reward": 0.5821865797042847,
      "step": 149
    },
    {
      "completion_length": 277.59376525878906,
      "epoch": 0.4,
      "grad_norm": 0.808658829177956,
      "kl": 0.002712249755859375,
      "learning_rate": 2e-06,
      "loss": 0.0533,
      "reward": 0.6419509947299957,
      "reward_std": 0.11133093386888504,
      "rewards/length_reward": 0.06517858430743217,
      "rewards/similarity_reward": 0.5767723917961121,
      "step": 150
    },
    {
      "completion_length": 284.6026916503906,
      "epoch": 0.4026666666666667,
      "grad_norm": 0.5726944777875907,
      "kl": 0.0017871856689453125,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.6313992738723755,
      "reward_std": 0.1251782327890396,
      "rewards/length_reward": 0.07008929550647736,
      "rewards/similarity_reward": 0.5613099485635757,
      "step": 151
    },
    {
      "completion_length": 269.39286041259766,
      "epoch": 0.4053333333333333,
      "grad_norm": 0.6397037674423297,
      "kl": 0.001956939697265625,
      "learning_rate": 2e-06,
      "loss": 0.0305,
      "reward": 0.6884091794490814,
      "reward_std": 0.08797129616141319,
      "rewards/length_reward": 0.07187500968575478,
      "rewards/similarity_reward": 0.6165341138839722,
      "step": 152
    },
    {
      "completion_length": 249.98214721679688,
      "epoch": 0.408,
      "grad_norm": 0.7056757365461955,
      "kl": 0.001522064208984375,
      "learning_rate": 2e-06,
      "loss": 0.051,
      "reward": 0.6953409612178802,
      "reward_std": 0.10321801900863647,
      "rewards/length_reward": 0.07321429625153542,
      "rewards/similarity_reward": 0.6221266388893127,
      "step": 153
    },
    {
      "completion_length": 301.4196472167969,
      "epoch": 0.4106666666666667,
      "grad_norm": 0.672677703608687,
      "kl": 0.001739501953125,
      "learning_rate": 2e-06,
      "loss": 0.0471,
      "reward": 0.6628727614879608,
      "reward_std": 0.09228364005684853,
      "rewards/length_reward": 0.06339287012815475,
      "rewards/similarity_reward": 0.5994798243045807,
      "step": 154
    },
    {
      "completion_length": 255.06697845458984,
      "epoch": 0.41333333333333333,
      "grad_norm": 0.7128891033714289,
      "kl": 0.00139617919921875,
      "learning_rate": 2e-06,
      "loss": 0.1005,
      "reward": 0.7157993614673615,
      "reward_std": 0.09617481008172035,
      "rewards/length_reward": 0.06339286454021931,
      "rewards/similarity_reward": 0.6524064540863037,
      "step": 155
    },
    {
      "completion_length": 283.53126525878906,
      "epoch": 0.416,
      "grad_norm": 0.6441555100001085,
      "kl": 0.00176239013671875,
      "learning_rate": 2e-06,
      "loss": 0.0274,
      "reward": 0.7534910440444946,
      "reward_std": 0.08639111369848251,
      "rewards/length_reward": 0.0781250037252903,
      "rewards/similarity_reward": 0.675366073846817,
      "step": 156
    },
    {
      "completion_length": 238.80805206298828,
      "epoch": 0.4186666666666667,
      "grad_norm": 0.6969348755424246,
      "kl": 0.002040863037109375,
      "learning_rate": 2e-06,
      "loss": 0.0883,
      "reward": 0.7300522923469543,
      "reward_std": 0.0814172811806202,
      "rewards/length_reward": 0.07991071790456772,
      "rewards/similarity_reward": 0.650141566991806,
      "step": 157
    },
    {
      "completion_length": 277.8526840209961,
      "epoch": 0.42133333333333334,
      "grad_norm": 0.6143618956533469,
      "kl": 0.002044677734375,
      "learning_rate": 2e-06,
      "loss": 0.0655,
      "reward": 0.6692302823066711,
      "reward_std": 0.11947489529848099,
      "rewards/length_reward": 0.07053572311997414,
      "rewards/similarity_reward": 0.5986945033073425,
      "step": 158
    },
    {
      "completion_length": 272.3526916503906,
      "epoch": 0.424,
      "grad_norm": 0.7167795024215189,
      "kl": 0.00182342529296875,
      "learning_rate": 2e-06,
      "loss": 0.0687,
      "reward": 0.6641163527965546,
      "reward_std": 0.11917684972286224,
      "rewards/length_reward": 0.0625000111758709,
      "rewards/similarity_reward": 0.6016163527965546,
      "step": 159
    },
    {
      "completion_length": 248.9821548461914,
      "epoch": 0.4266666666666667,
      "grad_norm": 0.7327545151839033,
      "kl": 0.00356292724609375,
      "learning_rate": 2e-06,
      "loss": 0.0745,
      "reward": 0.6552646160125732,
      "reward_std": 0.11251871287822723,
      "rewards/length_reward": 0.06294644065201283,
      "rewards/similarity_reward": 0.5923182666301727,
      "step": 160
    },
    {
      "completion_length": 256.07591247558594,
      "epoch": 0.42933333333333334,
      "grad_norm": 0.7535276542214958,
      "kl": 0.0027008056640625,
      "learning_rate": 2e-06,
      "loss": 0.0314,
      "reward": 0.6351377367973328,
      "reward_std": 0.10685055330395699,
      "rewards/length_reward": 0.059375012293457985,
      "rewards/similarity_reward": 0.5757627189159393,
      "step": 161
    },
    {
      "completion_length": 257.43750762939453,
      "epoch": 0.432,
      "grad_norm": 0.6595271125555469,
      "kl": 0.00250244140625,
      "learning_rate": 2e-06,
      "loss": -0.0056,
      "reward": 0.6979295313358307,
      "reward_std": 0.09611150622367859,
      "rewards/length_reward": 0.07187500968575478,
      "rewards/similarity_reward": 0.6260544955730438,
      "step": 162
    },
    {
      "completion_length": 262.4419860839844,
      "epoch": 0.43466666666666665,
      "grad_norm": 0.6185607216367858,
      "kl": 0.002216339111328125,
      "learning_rate": 2e-06,
      "loss": 0.0996,
      "reward": 0.6647298634052277,
      "reward_std": 0.15266689658164978,
      "rewards/length_reward": 0.06964286416769028,
      "rewards/similarity_reward": 0.5950870215892792,
      "step": 163
    },
    {
      "completion_length": 307.3526916503906,
      "epoch": 0.43733333333333335,
      "grad_norm": 0.7121773803870707,
      "kl": 0.001800537109375,
      "learning_rate": 2e-06,
      "loss": 0.0489,
      "reward": 0.7000181972980499,
      "reward_std": 0.12559669092297554,
      "rewards/length_reward": 0.06741072237491608,
      "rewards/similarity_reward": 0.6326074600219727,
      "step": 164
    },
    {
      "completion_length": 246.79019165039062,
      "epoch": 0.44,
      "grad_norm": 0.6721019148616484,
      "kl": 0.00197601318359375,
      "learning_rate": 2e-06,
      "loss": 0.0431,
      "reward": 0.7078234553337097,
      "reward_std": 0.09047579020261765,
      "rewards/length_reward": 0.06741072610020638,
      "rewards/similarity_reward": 0.6404128074645996,
      "step": 165
    },
    {
      "completion_length": 272.7723388671875,
      "epoch": 0.44266666666666665,
      "grad_norm": 0.5673761372545018,
      "kl": 0.002292633056640625,
      "learning_rate": 2e-06,
      "loss": 0.0246,
      "reward": 0.7385044991970062,
      "reward_std": 0.07343112863600254,
      "rewards/length_reward": 0.0714285783469677,
      "rewards/similarity_reward": 0.6670758426189423,
      "step": 166
    },
    {
      "completion_length": 236.49108123779297,
      "epoch": 0.44533333333333336,
      "grad_norm": 0.7731297375677283,
      "kl": 0.00337982177734375,
      "learning_rate": 2e-06,
      "loss": 0.0703,
      "reward": 0.6287428140640259,
      "reward_std": 0.13660584390163422,
      "rewards/length_reward": 0.058928582817316055,
      "rewards/similarity_reward": 0.5698141753673553,
      "step": 167
    },
    {
      "completion_length": 250.63839721679688,
      "epoch": 0.448,
      "grad_norm": 0.6844249610340352,
      "kl": 0.002716064453125,
      "learning_rate": 2e-06,
      "loss": 0.005,
      "reward": 0.6882721185684204,
      "reward_std": 0.08635392412543297,
      "rewards/length_reward": 0.07276786863803864,
      "rewards/similarity_reward": 0.6155042350292206,
      "step": 168
    },
    {
      "completion_length": 299.9151916503906,
      "epoch": 0.45066666666666666,
      "grad_norm": 0.5305327006388908,
      "kl": 0.001537322998046875,
      "learning_rate": 2e-06,
      "loss": 0.009,
      "reward": 0.7003590762615204,
      "reward_std": 0.06643648073077202,
      "rewards/length_reward": 0.0781250074505806,
      "rewards/similarity_reward": 0.6222340762615204,
      "step": 169
    },
    {
      "completion_length": 274.7143096923828,
      "epoch": 0.4533333333333333,
      "grad_norm": 0.6964990599197315,
      "kl": 0.002025604248046875,
      "learning_rate": 2e-06,
      "loss": 0.0422,
      "reward": 0.6924863159656525,
      "reward_std": 0.10944930091500282,
      "rewards/length_reward": 0.06830358505249023,
      "rewards/similarity_reward": 0.6241826713085175,
      "step": 170
    },
    {
      "completion_length": 260.29911041259766,
      "epoch": 0.456,
      "grad_norm": 0.7007252255586068,
      "kl": 0.0022430419921875,
      "learning_rate": 2e-06,
      "loss": 0.0892,
      "reward": 0.6610677242279053,
      "reward_std": 0.08702437207102776,
      "rewards/length_reward": 0.0625000111758709,
      "rewards/similarity_reward": 0.5985676944255829,
      "step": 171
    },
    {
      "completion_length": 291.8883972167969,
      "epoch": 0.45866666666666667,
      "grad_norm": 0.6818242026608151,
      "kl": 0.00199127197265625,
      "learning_rate": 2e-06,
      "loss": 0.0296,
      "reward": 0.6590520441532135,
      "reward_std": 0.11536738649010658,
      "rewards/length_reward": 0.058035723865032196,
      "rewards/similarity_reward": 0.6010163724422455,
      "step": 172
    },
    {
      "completion_length": 262.5982360839844,
      "epoch": 0.4613333333333333,
      "grad_norm": 0.7058845306121532,
      "kl": 0.00278472900390625,
      "learning_rate": 2e-06,
      "loss": 0.0147,
      "reward": 0.6392101943492889,
      "reward_std": 0.112693652510643,
      "rewards/length_reward": 0.0736607238650322,
      "rewards/similarity_reward": 0.5655494332313538,
      "step": 173
    },
    {
      "completion_length": 243.60269165039062,
      "epoch": 0.464,
      "grad_norm": 0.6664009632817656,
      "kl": 0.00189208984375,
      "learning_rate": 2e-06,
      "loss": -0.0054,
      "reward": 0.6843527555465698,
      "reward_std": 0.1105504259467125,
      "rewards/length_reward": 0.06964286416769028,
      "rewards/similarity_reward": 0.6147099137306213,
      "step": 174
    },
    {
      "completion_length": 250.46875762939453,
      "epoch": 0.4666666666666667,
      "grad_norm": 0.6473092015687406,
      "kl": 0.00228118896484375,
      "learning_rate": 2e-06,
      "loss": 0.0463,
      "reward": 0.6191678941249847,
      "reward_std": 0.1144786849617958,
      "rewards/length_reward": 0.06294644251465797,
      "rewards/similarity_reward": 0.5562214553356171,
      "step": 175
    },
    {
      "completion_length": 246.2276840209961,
      "epoch": 0.4693333333333333,
      "grad_norm": 0.782293454462268,
      "kl": 0.00267791748046875,
      "learning_rate": 2e-06,
      "loss": 0.104,
      "reward": 0.6215286254882812,
      "reward_std": 0.10633666813373566,
      "rewards/length_reward": 0.0691964402794838,
      "rewards/similarity_reward": 0.552332192659378,
      "step": 176
    },
    {
      "completion_length": 288.9598388671875,
      "epoch": 0.472,
      "grad_norm": 0.6068876990848109,
      "kl": 0.0028076171875,
      "learning_rate": 2e-06,
      "loss": 0.0499,
      "reward": 0.6609079539775848,
      "reward_std": 0.12471455708146095,
      "rewards/length_reward": 0.06294644251465797,
      "rewards/similarity_reward": 0.5979615151882172,
      "step": 177
    },
    {
      "completion_length": 231.6919708251953,
      "epoch": 0.4746666666666667,
      "grad_norm": 0.6072574366547386,
      "kl": 0.00218963623046875,
      "learning_rate": 2e-06,
      "loss": 0.0049,
      "reward": 0.6936376392841339,
      "reward_std": 0.09063273295760155,
      "rewards/length_reward": 0.0691964402794838,
      "rewards/similarity_reward": 0.6244412362575531,
      "step": 178
    },
    {
      "completion_length": 244.82144165039062,
      "epoch": 0.47733333333333333,
      "grad_norm": 0.6731140452564113,
      "kl": 0.002288818359375,
      "learning_rate": 2e-06,
      "loss": 0.0199,
      "reward": 0.6732902526855469,
      "reward_std": 0.12003319337964058,
      "rewards/length_reward": 0.06741072982549667,
      "rewards/similarity_reward": 0.6058795154094696,
      "step": 179
    },
    {
      "completion_length": 248.89733123779297,
      "epoch": 0.48,
      "grad_norm": 0.6436090869352918,
      "kl": 0.00261688232421875,
      "learning_rate": 2e-06,
      "loss": 0.0504,
      "reward": 0.6384674608707428,
      "reward_std": 0.13010089099407196,
      "rewards/length_reward": 0.057589298114180565,
      "rewards/similarity_reward": 0.5808781981468201,
      "step": 180
    },
    {
      "completion_length": 246.43304443359375,
      "epoch": 0.4826666666666667,
      "grad_norm": 0.6282260624355238,
      "kl": 0.005481719970703125,
      "learning_rate": 2e-06,
      "loss": 0.0045,
      "reward": 0.717792272567749,
      "reward_std": 0.07219249755144119,
      "rewards/length_reward": 0.07008929550647736,
      "rewards/similarity_reward": 0.6477029919624329,
      "step": 181
    },
    {
      "completion_length": 250.3660888671875,
      "epoch": 0.48533333333333334,
      "grad_norm": 0.8187805221092193,
      "kl": 0.00237274169921875,
      "learning_rate": 2e-06,
      "loss": 0.0499,
      "reward": 0.6446585357189178,
      "reward_std": 0.1213221587240696,
      "rewards/length_reward": 0.071428582072258,
      "rewards/similarity_reward": 0.573229968547821,
      "step": 182
    },
    {
      "completion_length": 238.0535888671875,
      "epoch": 0.488,
      "grad_norm": 0.7872695013078711,
      "kl": 0.0045166015625,
      "learning_rate": 2e-06,
      "loss": 0.0325,
      "reward": 0.6350027918815613,
      "reward_std": 0.08915703371167183,
      "rewards/length_reward": 0.06205358728766441,
      "rewards/similarity_reward": 0.5729491412639618,
      "step": 183
    },
    {
      "completion_length": 239.0446548461914,
      "epoch": 0.49066666666666664,
      "grad_norm": 0.6850244703959089,
      "kl": 0.002655029296875,
      "learning_rate": 2e-06,
      "loss": 0.0088,
      "reward": 0.6566590070724487,
      "reward_std": 0.08741006627678871,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.5901412069797516,
      "step": 184
    },
    {
      "completion_length": 228.43304443359375,
      "epoch": 0.49333333333333335,
      "grad_norm": 0.640590428870281,
      "kl": 0.00246429443359375,
      "learning_rate": 2e-06,
      "loss": 0.0211,
      "reward": 0.7332558631896973,
      "reward_std": 0.08987564593553543,
      "rewards/length_reward": 0.07098215445876122,
      "rewards/similarity_reward": 0.6622737348079681,
      "step": 185
    },
    {
      "completion_length": 275.46875762939453,
      "epoch": 0.496,
      "grad_norm": 0.7034238309978142,
      "kl": 0.00243377685546875,
      "learning_rate": 2e-06,
      "loss": 0.0345,
      "reward": 0.7175212800502777,
      "reward_std": 0.08901329711079597,
      "rewards/length_reward": 0.07544643431901932,
      "rewards/similarity_reward": 0.6420748233795166,
      "step": 186
    },
    {
      "completion_length": 280.00000762939453,
      "epoch": 0.49866666666666665,
      "grad_norm": 0.657474847366372,
      "kl": 0.00211334228515625,
      "learning_rate": 2e-06,
      "loss": 0.0942,
      "reward": 0.702696293592453,
      "reward_std": 0.08004150912165642,
      "rewards/length_reward": 0.0669642984867096,
      "rewards/similarity_reward": 0.635731965303421,
      "step": 187
    },
    {
      "completion_length": 246.83929443359375,
      "epoch": 0.5013333333333333,
      "grad_norm": 0.6840180489866146,
      "kl": 0.002269744873046875,
      "learning_rate": 2e-06,
      "loss": 0.0305,
      "reward": 0.6730459034442902,
      "reward_std": 0.08726384490728378,
      "rewards/length_reward": 0.07053572684526443,
      "rewards/similarity_reward": 0.6025101542472839,
      "step": 188
    },
    {
      "completion_length": 253.83929443359375,
      "epoch": 0.504,
      "grad_norm": 0.5974802952790658,
      "kl": 0.00257110595703125,
      "learning_rate": 2e-06,
      "loss": 0.0762,
      "reward": 0.6664688587188721,
      "reward_std": 0.11599006876349449,
      "rewards/length_reward": 0.07723214477300644,
      "rewards/similarity_reward": 0.5892367660999298,
      "step": 189
    },
    {
      "completion_length": 278.80804443359375,
      "epoch": 0.5066666666666667,
      "grad_norm": 0.6466637537752649,
      "kl": 0.00311279296875,
      "learning_rate": 2e-06,
      "loss": 0.0959,
      "reward": 0.6832419335842133,
      "reward_std": 0.11445752903819084,
      "rewards/length_reward": 0.06830358505249023,
      "rewards/similarity_reward": 0.614938348531723,
      "step": 190
    },
    {
      "completion_length": 235.5491180419922,
      "epoch": 0.5093333333333333,
      "grad_norm": 0.7153062964233342,
      "kl": 0.0019378662109375,
      "learning_rate": 2e-06,
      "loss": -0.0045,
      "reward": 0.6687261462211609,
      "reward_std": 0.09642783552408218,
      "rewards/length_reward": 0.06562500447034836,
      "rewards/similarity_reward": 0.6031011044979095,
      "step": 191
    },
    {
      "completion_length": 252.80358123779297,
      "epoch": 0.512,
      "grad_norm": 0.6313047035707975,
      "kl": 0.00281524658203125,
      "learning_rate": 2e-06,
      "loss": 0.0303,
      "reward": 0.6900187730789185,
      "reward_std": 0.10082340613007545,
      "rewards/length_reward": 0.07008929923176765,
      "rewards/similarity_reward": 0.6199295520782471,
      "step": 192
    },
    {
      "completion_length": 297.0580596923828,
      "epoch": 0.5146666666666667,
      "grad_norm": 0.6070377598000647,
      "kl": 0.001934051513671875,
      "learning_rate": 2e-06,
      "loss": 0.0237,
      "reward": 0.7174617052078247,
      "reward_std": 0.10485106706619263,
      "rewards/length_reward": 0.07544643804430962,
      "rewards/similarity_reward": 0.642015278339386,
      "step": 193
    },
    {
      "completion_length": 235.87500762939453,
      "epoch": 0.5173333333333333,
      "grad_norm": 0.6962391024556379,
      "kl": 0.00244140625,
      "learning_rate": 2e-06,
      "loss": 0.0154,
      "reward": 0.64629727602005,
      "reward_std": 0.0772336795926094,
      "rewards/length_reward": 0.07008929550647736,
      "rewards/similarity_reward": 0.5762079358100891,
      "step": 194
    },
    {
      "completion_length": 206.8482208251953,
      "epoch": 0.52,
      "grad_norm": 0.8558133302744291,
      "kl": 0.00318145751953125,
      "learning_rate": 2e-06,
      "loss": 0.0683,
      "reward": 0.6385850310325623,
      "reward_std": 0.0856214202940464,
      "rewards/length_reward": 0.06517857871949673,
      "rewards/similarity_reward": 0.5734064280986786,
      "step": 195
    },
    {
      "completion_length": 306.2633972167969,
      "epoch": 0.5226666666666666,
      "grad_norm": 0.5439460695909182,
      "kl": 0.0030517578125,
      "learning_rate": 2e-06,
      "loss": 0.0583,
      "reward": 0.6826609075069427,
      "reward_std": 0.08685707673430443,
      "rewards/length_reward": 0.07857143133878708,
      "rewards/similarity_reward": 0.6040894687175751,
      "step": 196
    },
    {
      "completion_length": 284.8169708251953,
      "epoch": 0.5253333333333333,
      "grad_norm": 0.597796232109364,
      "kl": 0.00238800048828125,
      "learning_rate": 2e-06,
      "loss": 0.0081,
      "reward": 0.7354407906532288,
      "reward_std": 0.06834723800420761,
      "rewards/length_reward": 0.07500000670552254,
      "rewards/similarity_reward": 0.6604407727718353,
      "step": 197
    },
    {
      "completion_length": 271.2901916503906,
      "epoch": 0.528,
      "grad_norm": 0.7779344829533027,
      "kl": 0.00433349609375,
      "learning_rate": 2e-06,
      "loss": 0.1361,
      "reward": 0.6831106841564178,
      "reward_std": 0.16374623030424118,
      "rewards/length_reward": 0.06875000894069672,
      "rewards/similarity_reward": 0.6143606305122375,
      "step": 198
    },
    {
      "completion_length": 257.1294708251953,
      "epoch": 0.5306666666666666,
      "grad_norm": 0.6115200963053203,
      "kl": 0.002285003662109375,
      "learning_rate": 2e-06,
      "loss": 0.0286,
      "reward": 0.6817789673805237,
      "reward_std": 0.07882888615131378,
      "rewards/length_reward": 0.07857143133878708,
      "rewards/similarity_reward": 0.6032074391841888,
      "step": 199
    },
    {
      "completion_length": 232.77679443359375,
      "epoch": 0.5333333333333333,
      "grad_norm": 0.7171528103086203,
      "kl": 0.0023651123046875,
      "learning_rate": 2e-06,
      "loss": 0.0367,
      "reward": 0.7090138792991638,
      "reward_std": 0.1117062047123909,
      "rewards/length_reward": 0.06741072610020638,
      "rewards/similarity_reward": 0.6416031718254089,
      "step": 200
    },
    {
      "completion_length": 292.0446472167969,
      "epoch": 0.536,
      "grad_norm": 0.6795728974282024,
      "kl": 0.0023193359375,
      "learning_rate": 2e-06,
      "loss": 0.0438,
      "reward": 0.6276125609874725,
      "reward_std": 0.09431122243404388,
      "rewards/length_reward": 0.06875000894069672,
      "rewards/similarity_reward": 0.5588625073432922,
      "step": 201
    },
    {
      "completion_length": 211.61161041259766,
      "epoch": 0.5386666666666666,
      "grad_norm": 0.9251828325921423,
      "kl": 0.0043487548828125,
      "learning_rate": 2e-06,
      "loss": 0.0439,
      "reward": 0.5786231458187103,
      "reward_std": 0.1044333316385746,
      "rewards/length_reward": 0.058928582817316055,
      "rewards/similarity_reward": 0.5196945518255234,
      "step": 202
    },
    {
      "completion_length": 241.27680206298828,
      "epoch": 0.5413333333333333,
      "grad_norm": 0.6521091947128457,
      "kl": 0.00319671630859375,
      "learning_rate": 2e-06,
      "loss": 0.0361,
      "reward": 0.6666609346866608,
      "reward_std": 0.092064518481493,
      "rewards/length_reward": 0.07544643431901932,
      "rewards/similarity_reward": 0.5912144482135773,
      "step": 203
    },
    {
      "completion_length": 211.28125762939453,
      "epoch": 0.544,
      "grad_norm": 0.6743614543760093,
      "kl": 0.00234222412109375,
      "learning_rate": 2e-06,
      "loss": 0.0135,
      "reward": 0.6388188004493713,
      "reward_std": 0.11344650015234947,
      "rewards/length_reward": 0.06785715371370316,
      "rewards/similarity_reward": 0.570961594581604,
      "step": 204
    },
    {
      "completion_length": 279.90179443359375,
      "epoch": 0.5466666666666666,
      "grad_norm": 0.6332153076352647,
      "kl": 0.00287628173828125,
      "learning_rate": 2e-06,
      "loss": 0.0423,
      "reward": 0.655907928943634,
      "reward_std": 0.0848810151219368,
      "rewards/length_reward": 0.07187500968575478,
      "rewards/similarity_reward": 0.5840329229831696,
      "step": 205
    },
    {
      "completion_length": 201.8169708251953,
      "epoch": 0.5493333333333333,
      "grad_norm": 0.85099814384088,
      "kl": 0.00351715087890625,
      "learning_rate": 2e-06,
      "loss": 0.0037,
      "reward": 0.5818662643432617,
      "reward_std": 0.09155267104506493,
      "rewards/length_reward": 0.058482155203819275,
      "rewards/similarity_reward": 0.5233840942382812,
      "step": 206
    },
    {
      "completion_length": 265.27679443359375,
      "epoch": 0.552,
      "grad_norm": 0.6261335014996906,
      "kl": 0.00209808349609375,
      "learning_rate": 2e-06,
      "loss": 0.0181,
      "reward": 0.7043133974075317,
      "reward_std": 0.09633800387382507,
      "rewards/length_reward": 0.06875001266598701,
      "rewards/similarity_reward": 0.6355634033679962,
      "step": 207
    },
    {
      "completion_length": 280.0669860839844,
      "epoch": 0.5546666666666666,
      "grad_norm": 0.6804923498664105,
      "kl": 0.00217437744140625,
      "learning_rate": 2e-06,
      "loss": 0.0146,
      "reward": 0.6569553017616272,
      "reward_std": 0.08773703873157501,
      "rewards/length_reward": 0.071428582072258,
      "rewards/similarity_reward": 0.585526704788208,
      "step": 208
    },
    {
      "completion_length": 274.3482208251953,
      "epoch": 0.5573333333333333,
      "grad_norm": 0.7277401805105727,
      "kl": 0.0023345947265625,
      "learning_rate": 2e-06,
      "loss": 0.1005,
      "reward": 0.7013810873031616,
      "reward_std": 0.09931684285402298,
      "rewards/length_reward": 0.07455357909202576,
      "rewards/similarity_reward": 0.6268274784088135,
      "step": 209
    },
    {
      "completion_length": 260.08036041259766,
      "epoch": 0.56,
      "grad_norm": 0.7807615068386718,
      "kl": 0.002655029296875,
      "learning_rate": 2e-06,
      "loss": 0.0773,
      "reward": 0.6310366690158844,
      "reward_std": 0.10352174565196037,
      "rewards/length_reward": 0.062053579837083817,
      "rewards/similarity_reward": 0.5689830929040909,
      "step": 210
    },
    {
      "completion_length": 259.75447845458984,
      "epoch": 0.5626666666666666,
      "grad_norm": 0.7093685270251532,
      "kl": 0.0039825439453125,
      "learning_rate": 2e-06,
      "loss": 0.0714,
      "reward": 0.6279307305812836,
      "reward_std": 0.10738151893019676,
      "rewards/length_reward": 0.06607143767178059,
      "rewards/similarity_reward": 0.5618593096733093,
      "step": 211
    },
    {
      "completion_length": 219.70983123779297,
      "epoch": 0.5653333333333334,
      "grad_norm": 0.8108428594239084,
      "kl": 0.00278472900390625,
      "learning_rate": 2e-06,
      "loss": 0.0942,
      "reward": 0.6396916508674622,
      "reward_std": 0.10166310518980026,
      "rewards/length_reward": 0.06785715371370316,
      "rewards/similarity_reward": 0.5718345046043396,
      "step": 212
    },
    {
      "completion_length": 232.99555206298828,
      "epoch": 0.568,
      "grad_norm": 0.6621005503481664,
      "kl": 0.00279998779296875,
      "learning_rate": 2e-06,
      "loss": -0.0055,
      "reward": 0.6306145489215851,
      "reward_std": 0.06187950074672699,
      "rewards/length_reward": 0.058482153341174126,
      "rewards/similarity_reward": 0.5721323788166046,
      "step": 213
    },
    {
      "completion_length": 252.12947845458984,
      "epoch": 0.5706666666666667,
      "grad_norm": 0.5674408374580422,
      "kl": 0.00339508056640625,
      "learning_rate": 2e-06,
      "loss": 0.0245,
      "reward": 0.6192521750926971,
      "reward_std": 0.07048613205552101,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.5527342855930328,
      "step": 214
    },
    {
      "completion_length": 255.9732208251953,
      "epoch": 0.5733333333333334,
      "grad_norm": 0.6904920555987271,
      "kl": 0.00270843505859375,
      "learning_rate": 2e-06,
      "loss": 0.0648,
      "reward": 0.6510066092014313,
      "reward_std": 0.056849006563425064,
      "rewards/length_reward": 0.0669642947614193,
      "rewards/similarity_reward": 0.5840422213077545,
      "step": 215
    },
    {
      "completion_length": 259.75001525878906,
      "epoch": 0.576,
      "grad_norm": 0.7131778652424658,
      "kl": 0.00354766845703125,
      "learning_rate": 2e-06,
      "loss": 0.0514,
      "reward": 0.6677349805831909,
      "reward_std": 0.10058147087693214,
      "rewards/length_reward": 0.058035725727677345,
      "rewards/similarity_reward": 0.6096992194652557,
      "step": 216
    },
    {
      "completion_length": 259.1294708251953,
      "epoch": 0.5786666666666667,
      "grad_norm": 0.5695386420901992,
      "kl": 0.00286102294921875,
      "learning_rate": 2e-06,
      "loss": 0.0224,
      "reward": 0.6971063017845154,
      "reward_std": 0.09981782734394073,
      "rewards/length_reward": 0.06741072610020638,
      "rewards/similarity_reward": 0.6296955347061157,
      "step": 217
    },
    {
      "completion_length": 236.69644927978516,
      "epoch": 0.5813333333333334,
      "grad_norm": 0.8176974415818429,
      "kl": 0.00765228271484375,
      "learning_rate": 2e-06,
      "loss": 0.0594,
      "reward": 0.611341118812561,
      "reward_std": 0.10692798718810081,
      "rewards/length_reward": 0.06428572721779346,
      "rewards/similarity_reward": 0.5470553934574127,
      "step": 218
    },
    {
      "completion_length": 251.71875762939453,
      "epoch": 0.584,
      "grad_norm": 0.5850098610329177,
      "kl": 0.002689361572265625,
      "learning_rate": 2e-06,
      "loss": 0.0162,
      "reward": 0.7100738286972046,
      "reward_std": 0.08857924118638039,
      "rewards/length_reward": 0.0714285783469677,
      "rewards/similarity_reward": 0.638645201921463,
      "step": 219
    },
    {
      "completion_length": 263.6875,
      "epoch": 0.5866666666666667,
      "grad_norm": 0.5616868949162326,
      "kl": 0.00293731689453125,
      "learning_rate": 2e-06,
      "loss": 0.014,
      "reward": 0.7153567671775818,
      "reward_std": 0.1150759868323803,
      "rewards/length_reward": 0.07857143506407738,
      "rewards/similarity_reward": 0.6367852687835693,
      "step": 220
    },
    {
      "completion_length": 259.24108123779297,
      "epoch": 0.5893333333333334,
      "grad_norm": 0.6551918580682736,
      "kl": 0.0029296875,
      "learning_rate": 2e-06,
      "loss": 0.015,
      "reward": 0.734772652387619,
      "reward_std": 0.07534275390207767,
      "rewards/length_reward": 0.07410714775323868,
      "rewards/similarity_reward": 0.6606654524803162,
      "step": 221
    },
    {
      "completion_length": 258.8303680419922,
      "epoch": 0.592,
      "grad_norm": 0.7150710475065644,
      "kl": 0.00344085693359375,
      "learning_rate": 2e-06,
      "loss": 0.0202,
      "reward": 0.6679186522960663,
      "reward_std": 0.10778569802641869,
      "rewards/length_reward": 0.06875000521540642,
      "rewards/similarity_reward": 0.5991686284542084,
      "step": 222
    },
    {
      "completion_length": 263.20091247558594,
      "epoch": 0.5946666666666667,
      "grad_norm": 0.6293755467193611,
      "kl": 0.00225067138671875,
      "learning_rate": 2e-06,
      "loss": 0.0133,
      "reward": 0.7472398281097412,
      "reward_std": 0.09251785278320312,
      "rewards/length_reward": 0.07723214849829674,
      "rewards/similarity_reward": 0.6700076460838318,
      "step": 223
    },
    {
      "completion_length": 270.21876525878906,
      "epoch": 0.5973333333333334,
      "grad_norm": 0.7206087593599326,
      "kl": 0.00225067138671875,
      "learning_rate": 2e-06,
      "loss": 0.1129,
      "reward": 0.6448234021663666,
      "reward_std": 0.13600966706871986,
      "rewards/length_reward": 0.060714298859238625,
      "rewards/similarity_reward": 0.5841090679168701,
      "step": 224
    },
    {
      "completion_length": 277.3839416503906,
      "epoch": 0.6,
      "grad_norm": 0.7555717124296486,
      "kl": 0.003326416015625,
      "learning_rate": 2e-06,
      "loss": 0.1094,
      "reward": 0.6888637244701385,
      "reward_std": 0.12441981956362724,
      "rewards/length_reward": 0.0669642947614193,
      "rewards/similarity_reward": 0.6218994557857513,
      "step": 225
    },
    {
      "completion_length": 288.4553680419922,
      "epoch": 0.6026666666666667,
      "grad_norm": 0.545561898466039,
      "kl": 0.00273895263671875,
      "learning_rate": 2e-06,
      "loss": 0.0106,
      "reward": 0.6741566956043243,
      "reward_std": 0.08976901695132256,
      "rewards/length_reward": 0.0825892873108387,
      "rewards/similarity_reward": 0.5915673971176147,
      "step": 226
    },
    {
      "completion_length": 291.31251525878906,
      "epoch": 0.6053333333333333,
      "grad_norm": 0.6862146524208239,
      "kl": 0.00433349609375,
      "learning_rate": 2e-06,
      "loss": 0.0199,
      "reward": 0.7247207760810852,
      "reward_std": 0.10038316994905472,
      "rewards/length_reward": 0.0691964402794838,
      "rewards/similarity_reward": 0.6555242836475372,
      "step": 227
    },
    {
      "completion_length": 237.46429443359375,
      "epoch": 0.608,
      "grad_norm": 0.6514945534975937,
      "kl": 0.00385284423828125,
      "learning_rate": 2e-06,
      "loss": 0.0598,
      "reward": 0.639589786529541,
      "reward_std": 0.08173046633601189,
      "rewards/length_reward": 0.06517858058214188,
      "rewards/similarity_reward": 0.5744112133979797,
      "step": 228
    },
    {
      "completion_length": 258.8393020629883,
      "epoch": 0.6106666666666667,
      "grad_norm": 0.5587705250798694,
      "kl": 0.001895904541015625,
      "learning_rate": 2e-06,
      "loss": -0.0066,
      "reward": 0.7333488166332245,
      "reward_std": 0.07437730580568314,
      "rewards/length_reward": 0.07946429029107094,
      "rewards/similarity_reward": 0.6538845300674438,
      "step": 229
    },
    {
      "completion_length": 237.80358123779297,
      "epoch": 0.6133333333333333,
      "grad_norm": 0.7749006841963979,
      "kl": 0.00295257568359375,
      "learning_rate": 2e-06,
      "loss": 0.0066,
      "reward": 0.6269674301147461,
      "reward_std": 0.09678621962666512,
      "rewards/length_reward": 0.06562500819563866,
      "rewards/similarity_reward": 0.5613424628973007,
      "step": 230
    },
    {
      "completion_length": 234.6116180419922,
      "epoch": 0.616,
      "grad_norm": 0.8857992900076056,
      "kl": 0.002620697021484375,
      "learning_rate": 2e-06,
      "loss": 0.036,
      "reward": 0.6997538506984711,
      "reward_std": 0.08686410635709763,
      "rewards/length_reward": 0.0691964402794838,
      "rewards/similarity_reward": 0.6305573880672455,
      "step": 231
    },
    {
      "completion_length": 287.6339416503906,
      "epoch": 0.6186666666666667,
      "grad_norm": 0.7389133589246519,
      "kl": 0.00316619873046875,
      "learning_rate": 2e-06,
      "loss": 0.0551,
      "reward": 0.6583640575408936,
      "reward_std": 0.1155795156955719,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.5918461382389069,
      "step": 232
    },
    {
      "completion_length": 288.3482208251953,
      "epoch": 0.6213333333333333,
      "grad_norm": 0.5841788620661817,
      "kl": 0.003204345703125,
      "learning_rate": 2e-06,
      "loss": 0.0042,
      "reward": 0.6696169376373291,
      "reward_std": 0.09103860706090927,
      "rewards/length_reward": 0.07410714775323868,
      "rewards/similarity_reward": 0.5955097377300262,
      "step": 233
    },
    {
      "completion_length": 277.43304443359375,
      "epoch": 0.624,
      "grad_norm": 0.5987965468099127,
      "kl": 0.00360107421875,
      "learning_rate": 2e-06,
      "loss": 0.0078,
      "reward": 0.7439869344234467,
      "reward_std": 0.08124737069010735,
      "rewards/length_reward": 0.07991072162985802,
      "rewards/similarity_reward": 0.664076179265976,
      "step": 234
    },
    {
      "completion_length": 264.87501525878906,
      "epoch": 0.6266666666666667,
      "grad_norm": 0.6848960422881881,
      "kl": 0.002899169921875,
      "learning_rate": 2e-06,
      "loss": 0.044,
      "reward": 0.7305464148521423,
      "reward_std": 0.05242483504116535,
      "rewards/length_reward": 0.0691964440047741,
      "rewards/similarity_reward": 0.6613499820232391,
      "step": 235
    },
    {
      "completion_length": 282.09376525878906,
      "epoch": 0.6293333333333333,
      "grad_norm": 0.6580896863492091,
      "kl": 0.0034637451171875,
      "learning_rate": 2e-06,
      "loss": 0.0489,
      "reward": 0.671272337436676,
      "reward_std": 0.10606055706739426,
      "rewards/length_reward": 0.059375010430812836,
      "rewards/similarity_reward": 0.6118973195552826,
      "step": 236
    },
    {
      "completion_length": 255.2946548461914,
      "epoch": 0.632,
      "grad_norm": 0.6382622236431467,
      "kl": 0.00327301025390625,
      "learning_rate": 2e-06,
      "loss": 0.0185,
      "reward": 0.7024641931056976,
      "reward_std": 0.09816553816199303,
      "rewards/length_reward": 0.07187501341104507,
      "rewards/similarity_reward": 0.6305891871452332,
      "step": 237
    },
    {
      "completion_length": 290.4375,
      "epoch": 0.6346666666666667,
      "grad_norm": 0.6729725697653197,
      "kl": 0.00388336181640625,
      "learning_rate": 2e-06,
      "loss": 0.0427,
      "reward": 0.6770188510417938,
      "reward_std": 0.08811097219586372,
      "rewards/length_reward": 0.07232144102454185,
      "rewards/similarity_reward": 0.6046973466873169,
      "step": 238
    },
    {
      "completion_length": 232.1384048461914,
      "epoch": 0.6373333333333333,
      "grad_norm": 0.729361913476319,
      "kl": 0.0040283203125,
      "learning_rate": 2e-06,
      "loss": 0.0698,
      "reward": 0.6408596038818359,
      "reward_std": 0.10725216567516327,
      "rewards/length_reward": 0.06741072237491608,
      "rewards/similarity_reward": 0.5734489262104034,
      "step": 239
    },
    {
      "completion_length": 284.1964416503906,
      "epoch": 0.64,
      "grad_norm": 0.6823808496445949,
      "kl": 0.0027923583984375,
      "learning_rate": 2e-06,
      "loss": 0.0463,
      "reward": 0.6450429260730743,
      "reward_std": 0.10368061438202858,
      "rewards/length_reward": 0.0669642984867096,
      "rewards/similarity_reward": 0.57807856798172,
      "step": 240
    },
    {
      "completion_length": 260.84822845458984,
      "epoch": 0.6426666666666667,
      "grad_norm": 0.5701250379634731,
      "kl": 0.00244140625,
      "learning_rate": 2e-06,
      "loss": 0.025,
      "reward": 0.7130960524082184,
      "reward_std": 0.0966276079416275,
      "rewards/length_reward": 0.08125000074505806,
      "rewards/similarity_reward": 0.631846010684967,
      "step": 241
    },
    {
      "completion_length": 224.5669708251953,
      "epoch": 0.6453333333333333,
      "grad_norm": 0.8310327870734144,
      "kl": 0.0076446533203125,
      "learning_rate": 2e-06,
      "loss": 0.0555,
      "reward": 0.5563714802265167,
      "reward_std": 0.12328523397445679,
      "rewards/length_reward": 0.054464295506477356,
      "rewards/similarity_reward": 0.5019071996212006,
      "step": 242
    },
    {
      "completion_length": 248.47322845458984,
      "epoch": 0.648,
      "grad_norm": 0.8542415499139471,
      "kl": 0.00330352783203125,
      "learning_rate": 2e-06,
      "loss": 0.1058,
      "reward": 0.6085337102413177,
      "reward_std": 0.12796913087368011,
      "rewards/length_reward": 0.056696439161896706,
      "rewards/similarity_reward": 0.5518373250961304,
      "step": 243
    },
    {
      "completion_length": 230.2321548461914,
      "epoch": 0.6506666666666666,
      "grad_norm": 0.8214724780628195,
      "kl": 0.0033111572265625,
      "learning_rate": 2e-06,
      "loss": 0.0369,
      "reward": 0.6595875322818756,
      "reward_std": 0.07628211192786694,
      "rewards/length_reward": 0.0656250100582838,
      "rewards/similarity_reward": 0.5939625054597855,
      "step": 244
    },
    {
      "completion_length": 271.4419860839844,
      "epoch": 0.6533333333333333,
      "grad_norm": 0.5339131510223507,
      "kl": 0.0025787353515625,
      "learning_rate": 2e-06,
      "loss": 0.0016,
      "reward": 0.7837041914463043,
      "reward_std": 0.07424943707883358,
      "rewards/length_reward": 0.07857143506407738,
      "rewards/similarity_reward": 0.7051327228546143,
      "step": 245
    },
    {
      "completion_length": 211.97322845458984,
      "epoch": 0.656,
      "grad_norm": 0.7281589894604906,
      "kl": 0.00310516357421875,
      "learning_rate": 2e-06,
      "loss": 0.0412,
      "reward": 0.6736421883106232,
      "reward_std": 0.08142560347914696,
      "rewards/length_reward": 0.07455357909202576,
      "rewards/similarity_reward": 0.5990886092185974,
      "step": 246
    },
    {
      "completion_length": 244.30805206298828,
      "epoch": 0.6586666666666666,
      "grad_norm": 0.6881733301939125,
      "kl": 0.002948760986328125,
      "learning_rate": 2e-06,
      "loss": 0.0521,
      "reward": 0.7186852991580963,
      "reward_std": 0.07804353907704353,
      "rewards/length_reward": 0.07544643431901932,
      "rewards/similarity_reward": 0.6432388722896576,
      "step": 247
    },
    {
      "completion_length": 204.83929443359375,
      "epoch": 0.6613333333333333,
      "grad_norm": 0.776497059513588,
      "kl": 0.00371551513671875,
      "learning_rate": 2e-06,
      "loss": 0.0183,
      "reward": 0.7021584212779999,
      "reward_std": 0.10252076759934425,
      "rewards/length_reward": 0.06562500819563866,
      "rewards/similarity_reward": 0.6365334391593933,
      "step": 248
    },
    {
      "completion_length": 248.63839721679688,
      "epoch": 0.664,
      "grad_norm": 0.6263280793364865,
      "kl": 0.0031890869140625,
      "learning_rate": 2e-06,
      "loss": 0.0264,
      "reward": 0.7189670205116272,
      "reward_std": 0.07904865220189095,
      "rewards/length_reward": 0.07410715520381927,
      "rewards/similarity_reward": 0.6448598206043243,
      "step": 249
    },
    {
      "completion_length": 247.37501525878906,
      "epoch": 0.6666666666666666,
      "grad_norm": 0.7055287610856288,
      "kl": 0.0048828125,
      "learning_rate": 2e-06,
      "loss": 0.0909,
      "reward": 0.6185382604598999,
      "reward_std": 0.10694902017712593,
      "rewards/length_reward": 0.06517858430743217,
      "rewards/similarity_reward": 0.553359717130661,
      "step": 250
    },
    {
      "completion_length": 247.83482360839844,
      "epoch": 0.6693333333333333,
      "grad_norm": 0.6610447004218395,
      "kl": 0.0035247802734375,
      "learning_rate": 2e-06,
      "loss": 0.0112,
      "reward": 0.5839682221412659,
      "reward_std": 0.0852314792573452,
      "rewards/length_reward": 0.0647321529686451,
      "rewards/similarity_reward": 0.5192360877990723,
      "step": 251
    },
    {
      "completion_length": 247.12501525878906,
      "epoch": 0.672,
      "grad_norm": 0.6773638528799676,
      "kl": 0.00295257568359375,
      "learning_rate": 2e-06,
      "loss": 0.0296,
      "reward": 0.6655312776565552,
      "reward_std": 0.10929644852876663,
      "rewards/length_reward": 0.060267869383096695,
      "rewards/similarity_reward": 0.6052633821964264,
      "step": 252
    },
    {
      "completion_length": 221.02679443359375,
      "epoch": 0.6746666666666666,
      "grad_norm": 0.7655042728512782,
      "kl": 0.00439453125,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.6889495253562927,
      "reward_std": 0.10270040482282639,
      "rewards/length_reward": 0.07410715147852898,
      "rewards/similarity_reward": 0.6148423850536346,
      "step": 253
    },
    {
      "completion_length": 251.55804443359375,
      "epoch": 0.6773333333333333,
      "grad_norm": 0.6277014120461456,
      "kl": 0.00347900390625,
      "learning_rate": 2e-06,
      "loss": 0.0195,
      "reward": 0.6920295655727386,
      "reward_std": 0.07308394275605679,
      "rewards/length_reward": 0.07276786491274834,
      "rewards/similarity_reward": 0.6192616820335388,
      "step": 254
    },
    {
      "completion_length": 247.24108123779297,
      "epoch": 0.68,
      "grad_norm": 0.8278631058587137,
      "kl": 0.0052032470703125,
      "learning_rate": 2e-06,
      "loss": 0.0801,
      "reward": 0.5797223746776581,
      "reward_std": 0.14392751455307007,
      "rewards/length_reward": 0.05446429364383221,
      "rewards/similarity_reward": 0.5252580344676971,
      "step": 255
    },
    {
      "completion_length": 283.0178680419922,
      "epoch": 0.6826666666666666,
      "grad_norm": 0.6453549089492197,
      "kl": 0.002655029296875,
      "learning_rate": 2e-06,
      "loss": 0.0299,
      "reward": 0.7264284193515778,
      "reward_std": 0.07789020985364914,
      "rewards/length_reward": 0.06964286416769028,
      "rewards/similarity_reward": 0.6567855477333069,
      "step": 256
    },
    {
      "completion_length": 299.46875,
      "epoch": 0.6853333333333333,
      "grad_norm": 0.5750252920299316,
      "kl": 0.00243377685546875,
      "learning_rate": 2e-06,
      "loss": 0.0429,
      "reward": 0.7198934257030487,
      "reward_std": 0.06834491342306137,
      "rewards/length_reward": 0.07232143729925156,
      "rewards/similarity_reward": 0.6475720107555389,
      "step": 257
    },
    {
      "completion_length": 250.46429443359375,
      "epoch": 0.688,
      "grad_norm": 0.6989181650264392,
      "kl": 0.00299835205078125,
      "learning_rate": 2e-06,
      "loss": 0.0029,
      "reward": 0.6974795460700989,
      "reward_std": 0.08845745399594307,
      "rewards/length_reward": 0.0691964365541935,
      "rewards/similarity_reward": 0.6282830834388733,
      "step": 258
    },
    {
      "completion_length": 249.0848388671875,
      "epoch": 0.6906666666666667,
      "grad_norm": 0.768500388572952,
      "kl": 0.0030670166015625,
      "learning_rate": 2e-06,
      "loss": 0.052,
      "reward": 0.6331368982791901,
      "reward_std": 0.09397462010383606,
      "rewards/length_reward": 0.06741072237491608,
      "rewards/similarity_reward": 0.5657261312007904,
      "step": 259
    },
    {
      "completion_length": 205.52233123779297,
      "epoch": 0.6933333333333334,
      "grad_norm": 0.8190032734025428,
      "kl": 0.0068511962890625,
      "learning_rate": 2e-06,
      "loss": -0.0181,
      "reward": 0.6275625824928284,
      "reward_std": 0.0882430449128151,
      "rewards/length_reward": 0.062053583562374115,
      "rewards/similarity_reward": 0.5655089616775513,
      "step": 260
    },
    {
      "completion_length": 227.0134048461914,
      "epoch": 0.696,
      "grad_norm": 0.6819806218771027,
      "kl": 0.0030364990234375,
      "learning_rate": 2e-06,
      "loss": 0.0396,
      "reward": 0.7068421840667725,
      "reward_std": 0.0854947492480278,
      "rewards/length_reward": 0.06741072610020638,
      "rewards/similarity_reward": 0.63943150639534,
      "step": 261
    },
    {
      "completion_length": 277.65625762939453,
      "epoch": 0.6986666666666667,
      "grad_norm": 0.6041036479677462,
      "kl": 0.002208709716796875,
      "learning_rate": 2e-06,
      "loss": 0.0162,
      "reward": 0.7707177698612213,
      "reward_std": 0.060075582936406136,
      "rewards/length_reward": 0.08169642835855484,
      "rewards/similarity_reward": 0.6890212595462799,
      "step": 262
    },
    {
      "completion_length": 273.18750762939453,
      "epoch": 0.7013333333333334,
      "grad_norm": 0.6716948511043099,
      "kl": 0.00411224365234375,
      "learning_rate": 2e-06,
      "loss": 0.0791,
      "reward": 0.6893300712108612,
      "reward_std": 0.08914920315146446,
      "rewards/length_reward": 0.07098215073347092,
      "rewards/similarity_reward": 0.6183479726314545,
      "step": 263
    },
    {
      "completion_length": 195.03125762939453,
      "epoch": 0.704,
      "grad_norm": 0.7288219165266566,
      "kl": 0.0046539306640625,
      "learning_rate": 2e-06,
      "loss": 0.0365,
      "reward": 0.6311258971691132,
      "reward_std": 0.08954550698399544,
      "rewards/length_reward": 0.06741072610020638,
      "rewards/similarity_reward": 0.5637151300907135,
      "step": 264
    },
    {
      "completion_length": 230.5714340209961,
      "epoch": 0.7066666666666667,
      "grad_norm": 0.6590047515085815,
      "kl": 0.00440216064453125,
      "learning_rate": 2e-06,
      "loss": -0.0116,
      "reward": 0.6623952388763428,
      "reward_std": 0.07462666183710098,
      "rewards/length_reward": 0.06562501192092896,
      "rewards/similarity_reward": 0.5967701971530914,
      "step": 265
    },
    {
      "completion_length": 227.8616180419922,
      "epoch": 0.7093333333333334,
      "grad_norm": 0.6922227485667228,
      "kl": 0.00439453125,
      "learning_rate": 2e-06,
      "loss": 0.0636,
      "reward": 0.6782627999782562,
      "reward_std": 0.11519866064190865,
      "rewards/length_reward": 0.061607154086232185,
      "rewards/similarity_reward": 0.6166556179523468,
      "step": 266
    },
    {
      "completion_length": 257.0714416503906,
      "epoch": 0.712,
      "grad_norm": 0.5520470941775752,
      "kl": 0.00386810302734375,
      "learning_rate": 2e-06,
      "loss": 0.0104,
      "reward": 0.7550633549690247,
      "reward_std": 0.09709116071462631,
      "rewards/length_reward": 0.07008930295705795,
      "rewards/similarity_reward": 0.6849740147590637,
      "step": 267
    },
    {
      "completion_length": 232.32591247558594,
      "epoch": 0.7146666666666667,
      "grad_norm": 0.7138805826191377,
      "kl": 0.003936767578125,
      "learning_rate": 2e-06,
      "loss": 0.0331,
      "reward": 0.669314980506897,
      "reward_std": 0.09859981015324593,
      "rewards/length_reward": 0.06517858244478703,
      "rewards/similarity_reward": 0.6041364073753357,
      "step": 268
    },
    {
      "completion_length": 237.11162567138672,
      "epoch": 0.7173333333333334,
      "grad_norm": 0.6150700165348351,
      "kl": 0.00328826904296875,
      "learning_rate": 2e-06,
      "loss": 0.0028,
      "reward": 0.7752477526664734,
      "reward_std": 0.057502467185258865,
      "rewards/length_reward": 0.0781250037252903,
      "rewards/similarity_reward": 0.6971226334571838,
      "step": 269
    },
    {
      "completion_length": 254.75447845458984,
      "epoch": 0.72,
      "grad_norm": 0.7962475762237703,
      "kl": 0.00536346435546875,
      "learning_rate": 2e-06,
      "loss": 0.0608,
      "reward": 0.6644669771194458,
      "reward_std": 0.10142809525132179,
      "rewards/length_reward": 0.056250009685754776,
      "rewards/similarity_reward": 0.6082169711589813,
      "step": 270
    },
    {
      "completion_length": 263.02679443359375,
      "epoch": 0.7226666666666667,
      "grad_norm": 0.6290318550051345,
      "kl": 0.00321197509765625,
      "learning_rate": 2e-06,
      "loss": 0.0428,
      "reward": 0.6874348521232605,
      "reward_std": 0.08607123605906963,
      "rewards/length_reward": 0.0691964328289032,
      "rewards/similarity_reward": 0.6182384490966797,
      "step": 271
    },
    {
      "completion_length": 253.59375762939453,
      "epoch": 0.7253333333333334,
      "grad_norm": 0.5631283015221202,
      "kl": 0.0034027099609375,
      "learning_rate": 2e-06,
      "loss": -0.0079,
      "reward": 0.7192228138446808,
      "reward_std": 0.05196613632142544,
      "rewards/length_reward": 0.08392857387661934,
      "rewards/similarity_reward": 0.6352941989898682,
      "step": 272
    },
    {
      "completion_length": 275.14732360839844,
      "epoch": 0.728,
      "grad_norm": 0.5719219206376979,
      "kl": 0.00231170654296875,
      "learning_rate": 2e-06,
      "loss": 0.0147,
      "reward": 0.7571655213832855,
      "reward_std": 0.07733196392655373,
      "rewards/length_reward": 0.07678572088479996,
      "rewards/similarity_reward": 0.6803797781467438,
      "step": 273
    },
    {
      "completion_length": 235.5982208251953,
      "epoch": 0.7306666666666667,
      "grad_norm": 0.694430050013244,
      "kl": 0.0037841796875,
      "learning_rate": 2e-06,
      "loss": 0.0622,
      "reward": 0.7187261283397675,
      "reward_std": 0.11743078380823135,
      "rewards/length_reward": 0.0669642947614193,
      "rewards/similarity_reward": 0.6517618000507355,
      "step": 274
    },
    {
      "completion_length": 284.7678680419922,
      "epoch": 0.7333333333333333,
      "grad_norm": 0.5786627728822946,
      "kl": 0.00365447998046875,
      "learning_rate": 2e-06,
      "loss": 0.0085,
      "reward": 0.7249119877815247,
      "reward_std": 0.06474133767187595,
      "rewards/length_reward": 0.07633929327130318,
      "rewards/similarity_reward": 0.648572713136673,
      "step": 275
    },
    {
      "completion_length": 268.0580596923828,
      "epoch": 0.736,
      "grad_norm": 0.554118778302666,
      "kl": 0.0027008056640625,
      "learning_rate": 2e-06,
      "loss": 0.0133,
      "reward": 0.7421092092990875,
      "reward_std": 0.06751231849193573,
      "rewards/length_reward": 0.07544643431901932,
      "rewards/similarity_reward": 0.6666627824306488,
      "step": 276
    },
    {
      "completion_length": 191.9866180419922,
      "epoch": 0.7386666666666667,
      "grad_norm": 0.7484110778385685,
      "kl": 0.0081329345703125,
      "learning_rate": 2e-06,
      "loss": -0.0117,
      "reward": 0.5663988292217255,
      "reward_std": 0.11341952532529831,
      "rewards/length_reward": 0.055357156321406364,
      "rewards/similarity_reward": 0.5110416412353516,
      "step": 277
    },
    {
      "completion_length": 262.1428680419922,
      "epoch": 0.7413333333333333,
      "grad_norm": 0.6308341820211528,
      "kl": 0.003448486328125,
      "learning_rate": 2e-06,
      "loss": 0.0113,
      "reward": 0.6712258756160736,
      "reward_std": 0.10108800232410431,
      "rewards/length_reward": 0.0691964440047741,
      "rewards/similarity_reward": 0.6020293831825256,
      "step": 278
    },
    {
      "completion_length": 257.31251525878906,
      "epoch": 0.744,
      "grad_norm": 0.7444571736292193,
      "kl": 0.0075836181640625,
      "learning_rate": 2e-06,
      "loss": 0.0194,
      "reward": 0.6420714855194092,
      "reward_std": 0.10531632602214813,
      "rewards/length_reward": 0.06830358505249023,
      "rewards/similarity_reward": 0.5737679302692413,
      "step": 279
    },
    {
      "completion_length": 273.0714416503906,
      "epoch": 0.7466666666666667,
      "grad_norm": 0.6772210722840282,
      "kl": 0.0051116943359375,
      "learning_rate": 2e-06,
      "loss": 0.0147,
      "reward": 0.6341734230518341,
      "reward_std": 0.0923330970108509,
      "rewards/length_reward": 0.07321428880095482,
      "rewards/similarity_reward": 0.5609591007232666,
      "step": 280
    },
    {
      "completion_length": 270.4151840209961,
      "epoch": 0.7493333333333333,
      "grad_norm": 0.6993063816092533,
      "kl": 0.0049591064453125,
      "learning_rate": 2e-06,
      "loss": 0.0501,
      "reward": 0.6939212083816528,
      "reward_std": 0.10334672406315804,
      "rewards/length_reward": 0.0736607201397419,
      "rewards/similarity_reward": 0.6202605366706848,
      "step": 281
    },
    {
      "completion_length": 233.01339721679688,
      "epoch": 0.752,
      "grad_norm": 0.7285449434821649,
      "kl": 0.0037841796875,
      "learning_rate": 2e-06,
      "loss": 0.013,
      "reward": 0.6847032904624939,
      "reward_std": 0.06913377530872822,
      "rewards/length_reward": 0.06651786714792252,
      "rewards/similarity_reward": 0.618185430765152,
      "step": 282
    },
    {
      "completion_length": 238.3482208251953,
      "epoch": 0.7546666666666667,
      "grad_norm": 0.90960398931583,
      "kl": 0.0053253173828125,
      "learning_rate": 2e-06,
      "loss": 0.019,
      "reward": 0.5968413352966309,
      "reward_std": 0.12051042169332504,
      "rewards/length_reward": 0.058035727590322495,
      "rewards/similarity_reward": 0.5388056635856628,
      "step": 283
    },
    {
      "completion_length": 296.4821472167969,
      "epoch": 0.7573333333333333,
      "grad_norm": 0.6037765863353864,
      "kl": 0.004730224609375,
      "learning_rate": 2e-06,
      "loss": -0.0322,
      "reward": 0.6755504310131073,
      "reward_std": 0.06930392794311047,
      "rewards/length_reward": 0.07053572684526443,
      "rewards/similarity_reward": 0.6050147414207458,
      "step": 284
    },
    {
      "completion_length": 235.1339340209961,
      "epoch": 0.76,
      "grad_norm": 0.5442699124141502,
      "kl": 0.002349853515625,
      "learning_rate": 2e-06,
      "loss": -0.0114,
      "reward": 0.7083780765533447,
      "reward_std": 0.06875683926045895,
      "rewards/length_reward": 0.07633928954601288,
      "rewards/similarity_reward": 0.632038801908493,
      "step": 285
    },
    {
      "completion_length": 259.0401916503906,
      "epoch": 0.7626666666666667,
      "grad_norm": 0.7121048521613264,
      "kl": 0.00444793701171875,
      "learning_rate": 2e-06,
      "loss": 0.0064,
      "reward": 0.6680812537670135,
      "reward_std": 0.077706228941679,
      "rewards/length_reward": 0.06607143953442574,
      "rewards/similarity_reward": 0.6020097434520721,
      "step": 286
    },
    {
      "completion_length": 270.7366256713867,
      "epoch": 0.7653333333333333,
      "grad_norm": 0.6208584555138446,
      "kl": 0.003662109375,
      "learning_rate": 2e-06,
      "loss": 0.015,
      "reward": 0.706127256155014,
      "reward_std": 0.11445373296737671,
      "rewards/length_reward": 0.07053572684526443,
      "rewards/similarity_reward": 0.635591596364975,
      "step": 287
    },
    {
      "completion_length": 250.53573608398438,
      "epoch": 0.768,
      "grad_norm": 0.7385667089013174,
      "kl": 0.0044708251953125,
      "learning_rate": 2e-06,
      "loss": 0.032,
      "reward": 0.6525153517723083,
      "reward_std": 0.08994012698531151,
      "rewards/length_reward": 0.07053572311997414,
      "rewards/similarity_reward": 0.5819795727729797,
      "step": 288
    },
    {
      "completion_length": 271.4241180419922,
      "epoch": 0.7706666666666667,
      "grad_norm": 0.571768569159425,
      "kl": 0.00421142578125,
      "learning_rate": 2e-06,
      "loss": 0.0514,
      "reward": 0.6859732568264008,
      "reward_std": 0.10971596091985703,
      "rewards/length_reward": 0.07410715147852898,
      "rewards/similarity_reward": 0.6118661463260651,
      "step": 289
    },
    {
      "completion_length": 279.0089416503906,
      "epoch": 0.7733333333333333,
      "grad_norm": 0.5580184118619138,
      "kl": 0.00385284423828125,
      "learning_rate": 2e-06,
      "loss": 0.0172,
      "reward": 0.7247184813022614,
      "reward_std": 0.11090399697422981,
      "rewards/length_reward": 0.07232144102454185,
      "rewards/similarity_reward": 0.652397096157074,
      "step": 290
    },
    {
      "completion_length": 268.5357208251953,
      "epoch": 0.776,
      "grad_norm": 0.6568855951756748,
      "kl": 0.0045928955078125,
      "learning_rate": 2e-06,
      "loss": -0.0124,
      "reward": 0.6895658373832703,
      "reward_std": 0.09936371073126793,
      "rewards/length_reward": 0.07321429252624512,
      "rewards/similarity_reward": 0.6163516044616699,
      "step": 291
    },
    {
      "completion_length": 290.41072845458984,
      "epoch": 0.7786666666666666,
      "grad_norm": 0.5426211050313083,
      "kl": 0.0032196044921875,
      "learning_rate": 2e-06,
      "loss": 0.019,
      "reward": 0.7200377881526947,
      "reward_std": 0.06104239821434021,
      "rewards/length_reward": 0.07187500968575478,
      "rewards/similarity_reward": 0.6481626629829407,
      "step": 292
    },
    {
      "completion_length": 246.53572845458984,
      "epoch": 0.7813333333333333,
      "grad_norm": 0.7741354483614902,
      "kl": 0.0047454833984375,
      "learning_rate": 2e-06,
      "loss": 0.05,
      "reward": 0.6441958248615265,
      "reward_std": 0.13777245581150055,
      "rewards/length_reward": 0.06428572908043861,
      "rewards/similarity_reward": 0.5799101293087006,
      "step": 293
    },
    {
      "completion_length": 282.1339416503906,
      "epoch": 0.784,
      "grad_norm": 0.6078928600918315,
      "kl": 0.0030059814453125,
      "learning_rate": 2e-06,
      "loss": 0.0654,
      "reward": 0.7353447079658508,
      "reward_std": 0.11103275418281555,
      "rewards/length_reward": 0.0736607238650322,
      "rewards/similarity_reward": 0.6616839170455933,
      "step": 294
    },
    {
      "completion_length": 277.2723388671875,
      "epoch": 0.7866666666666666,
      "grad_norm": 0.6681511715620417,
      "kl": 0.0036773681640625,
      "learning_rate": 2e-06,
      "loss": 0.0328,
      "reward": 0.6674771308898926,
      "reward_std": 0.0996098667383194,
      "rewards/length_reward": 0.07321429252624512,
      "rewards/similarity_reward": 0.5942628383636475,
      "step": 295
    },
    {
      "completion_length": 239.18304443359375,
      "epoch": 0.7893333333333333,
      "grad_norm": 0.587894018606086,
      "kl": 0.0035858154296875,
      "learning_rate": 2e-06,
      "loss": 0.0416,
      "reward": 0.6579557359218597,
      "reward_std": 0.08152700588107109,
      "rewards/length_reward": 0.06919643469154835,
      "rewards/similarity_reward": 0.5887593328952789,
      "step": 296
    },
    {
      "completion_length": 234.94197845458984,
      "epoch": 0.792,
      "grad_norm": 0.6099426936791735,
      "kl": 0.0045318603515625,
      "learning_rate": 2e-06,
      "loss": 0.0133,
      "reward": 0.6847122013568878,
      "reward_std": 0.1407642886042595,
      "rewards/length_reward": 0.06830357573926449,
      "rewards/similarity_reward": 0.6164086163043976,
      "step": 297
    },
    {
      "completion_length": 238.58483123779297,
      "epoch": 0.7946666666666666,
      "grad_norm": 0.7568584099378661,
      "kl": 0.0032196044921875,
      "learning_rate": 2e-06,
      "loss": 0.0816,
      "reward": 0.7135740518569946,
      "reward_std": 0.08400523662567139,
      "rewards/length_reward": 0.07678572088479996,
      "rewards/similarity_reward": 0.63678839802742,
      "step": 298
    },
    {
      "completion_length": 265.5178680419922,
      "epoch": 0.7973333333333333,
      "grad_norm": 0.559632299831719,
      "kl": 0.00363922119140625,
      "learning_rate": 2e-06,
      "loss": 0.0075,
      "reward": 0.7292493581771851,
      "reward_std": 0.06625411659479141,
      "rewards/length_reward": 0.07544643431901932,
      "rewards/similarity_reward": 0.6538029313087463,
      "step": 299
    },
    {
      "completion_length": 266.7366256713867,
      "epoch": 0.8,
      "grad_norm": 0.6245003701307471,
      "kl": 0.00376129150390625,
      "learning_rate": 2e-06,
      "loss": 0.0309,
      "reward": 0.6834622323513031,
      "reward_std": 0.09874312952160835,
      "rewards/length_reward": 0.07053572684526443,
      "rewards/similarity_reward": 0.6129264831542969,
      "step": 300
    },
    {
      "completion_length": 268.4464416503906,
      "epoch": 0.8026666666666666,
      "grad_norm": 0.6001886392680107,
      "kl": 0.00341796875,
      "learning_rate": 2e-06,
      "loss": -0.0133,
      "reward": 0.7123071253299713,
      "reward_std": 0.05348840728402138,
      "rewards/length_reward": 0.06741072982549667,
      "rewards/similarity_reward": 0.6448963582515717,
      "step": 301
    },
    {
      "completion_length": 226.2321548461914,
      "epoch": 0.8053333333333333,
      "grad_norm": 0.6336093254745343,
      "kl": 0.00481414794921875,
      "learning_rate": 2e-06,
      "loss": 0.011,
      "reward": 0.6465486586093903,
      "reward_std": 0.09404471330344677,
      "rewards/length_reward": 0.06607143767178059,
      "rewards/similarity_reward": 0.5804772675037384,
      "step": 302
    },
    {
      "completion_length": 269.90179443359375,
      "epoch": 0.808,
      "grad_norm": 0.59634105150443,
      "kl": 0.00426483154296875,
      "learning_rate": 2e-06,
      "loss": 0.0036,
      "reward": 0.6335479617118835,
      "reward_std": 0.0826010163873434,
      "rewards/length_reward": 0.07500001415610313,
      "rewards/similarity_reward": 0.5585480332374573,
      "step": 303
    },
    {
      "completion_length": 277.3571472167969,
      "epoch": 0.8106666666666666,
      "grad_norm": 0.6002262528664385,
      "kl": 0.00386810302734375,
      "learning_rate": 2e-06,
      "loss": 0.081,
      "reward": 0.6462258398532867,
      "reward_std": 0.10935888066887856,
      "rewards/length_reward": 0.06875000894069672,
      "rewards/similarity_reward": 0.5774758458137512,
      "step": 304
    },
    {
      "completion_length": 291.7276916503906,
      "epoch": 0.8133333333333334,
      "grad_norm": 0.6664322809544898,
      "kl": 0.0032958984375,
      "learning_rate": 2e-06,
      "loss": 0.0743,
      "reward": 0.754040002822876,
      "reward_std": 0.07836959138512611,
      "rewards/length_reward": 0.07455357536673546,
      "rewards/similarity_reward": 0.6794863939285278,
      "step": 305
    },
    {
      "completion_length": 256.62501525878906,
      "epoch": 0.816,
      "grad_norm": 0.6918122711031781,
      "kl": 0.003997802734375,
      "learning_rate": 2e-06,
      "loss": 0.0019,
      "reward": 0.651542603969574,
      "reward_std": 0.11015984788537025,
      "rewards/length_reward": 0.06383929774165154,
      "rewards/similarity_reward": 0.587703287601471,
      "step": 306
    },
    {
      "completion_length": 269.5803680419922,
      "epoch": 0.8186666666666667,
      "grad_norm": 0.6835757950482498,
      "kl": 0.0058746337890625,
      "learning_rate": 2e-06,
      "loss": 0.0137,
      "reward": 0.6834548711776733,
      "reward_std": 0.09235357493162155,
      "rewards/length_reward": 0.06517858058214188,
      "rewards/similarity_reward": 0.6182762682437897,
      "step": 307
    },
    {
      "completion_length": 249.43751525878906,
      "epoch": 0.8213333333333334,
      "grad_norm": 0.6594580830657123,
      "kl": 0.00449371337890625,
      "learning_rate": 2e-06,
      "loss": -0.0019,
      "reward": 0.6838014125823975,
      "reward_std": 0.12604531273245811,
      "rewards/length_reward": 0.07008929550647736,
      "rewards/similarity_reward": 0.6137120425701141,
      "step": 308
    },
    {
      "completion_length": 272.4151916503906,
      "epoch": 0.824,
      "grad_norm": 0.616617412578935,
      "kl": 0.00392913818359375,
      "learning_rate": 2e-06,
      "loss": 0.0372,
      "reward": 0.7449055016040802,
      "reward_std": 0.08290699683129787,
      "rewards/length_reward": 0.07276786491274834,
      "rewards/similarity_reward": 0.6721376478672028,
      "step": 309
    },
    {
      "completion_length": 255.8660888671875,
      "epoch": 0.8266666666666667,
      "grad_norm": 0.6324884031752415,
      "kl": 0.00347900390625,
      "learning_rate": 2e-06,
      "loss": -0.014,
      "reward": 0.6722660958766937,
      "reward_std": 0.08029109984636307,
      "rewards/length_reward": 0.07232143729925156,
      "rewards/similarity_reward": 0.5999446511268616,
      "step": 310
    },
    {
      "completion_length": 229.05358123779297,
      "epoch": 0.8293333333333334,
      "grad_norm": 0.8069840909052901,
      "kl": 0.00518798828125,
      "learning_rate": 2e-06,
      "loss": 0.0471,
      "reward": 0.6889167726039886,
      "reward_std": 0.10098548233509064,
      "rewards/length_reward": 0.0714285783469677,
      "rewards/similarity_reward": 0.6174881756305695,
      "step": 311
    },
    {
      "completion_length": 244.28125762939453,
      "epoch": 0.832,
      "grad_norm": 0.6703576381079311,
      "kl": 0.00594329833984375,
      "learning_rate": 2e-06,
      "loss": 0.0115,
      "reward": 0.7172508239746094,
      "reward_std": 0.07987817749381065,
      "rewards/length_reward": 0.07008929178118706,
      "rewards/similarity_reward": 0.647161453962326,
      "step": 312
    },
    {
      "completion_length": 235.50000762939453,
      "epoch": 0.8346666666666667,
      "grad_norm": 0.6595218129039281,
      "kl": 0.00487518310546875,
      "learning_rate": 2e-06,
      "loss": -0.0028,
      "reward": 0.661961704492569,
      "reward_std": 0.09113451465964317,
      "rewards/length_reward": 0.06607143767178059,
      "rewards/similarity_reward": 0.5958903431892395,
      "step": 313
    },
    {
      "completion_length": 219.86608123779297,
      "epoch": 0.8373333333333334,
      "grad_norm": 0.7020826871224611,
      "kl": 0.0062713623046875,
      "learning_rate": 2e-06,
      "loss": 0.0162,
      "reward": 0.6874544024467468,
      "reward_std": 0.07596220076084137,
      "rewards/length_reward": 0.0647321566939354,
      "rewards/similarity_reward": 0.6227222084999084,
      "step": 314
    },
    {
      "completion_length": 264.8616180419922,
      "epoch": 0.84,
      "grad_norm": 0.586765871425098,
      "kl": 0.00507354736328125,
      "learning_rate": 2e-06,
      "loss": -0.0005,
      "reward": 0.7921656966209412,
      "reward_std": 0.0773557759821415,
      "rewards/length_reward": 0.0781250074505806,
      "rewards/similarity_reward": 0.7140407264232635,
      "step": 315
    },
    {
      "completion_length": 268.8348388671875,
      "epoch": 0.8426666666666667,
      "grad_norm": 0.556140838025419,
      "kl": 0.004425048828125,
      "learning_rate": 2e-06,
      "loss": 0.0045,
      "reward": 0.7406611740589142,
      "reward_std": 0.07832159847021103,
      "rewards/length_reward": 0.07232143729925156,
      "rewards/similarity_reward": 0.668339729309082,
      "step": 316
    },
    {
      "completion_length": 286.6696548461914,
      "epoch": 0.8453333333333334,
      "grad_norm": 0.6317684457245647,
      "kl": 0.00397491455078125,
      "learning_rate": 2e-06,
      "loss": 0.0177,
      "reward": 0.791156679391861,
      "reward_std": 0.06108024716377258,
      "rewards/length_reward": 0.0781250074505806,
      "rewards/similarity_reward": 0.7130315899848938,
      "step": 317
    },
    {
      "completion_length": 223.43305206298828,
      "epoch": 0.848,
      "grad_norm": 0.7967746076439991,
      "kl": 0.0052642822265625,
      "learning_rate": 2e-06,
      "loss": 0.0238,
      "reward": 0.7096598744392395,
      "reward_std": 0.08100546710193157,
      "rewards/length_reward": 0.07098214700818062,
      "rewards/similarity_reward": 0.6386776566505432,
      "step": 318
    },
    {
      "completion_length": 248.56697845458984,
      "epoch": 0.8506666666666667,
      "grad_norm": 0.6967301959645911,
      "kl": 0.004791259765625,
      "learning_rate": 2e-06,
      "loss": 0.0788,
      "reward": 0.712384045124054,
      "reward_std": 0.08104405552148819,
      "rewards/length_reward": 0.07633929327130318,
      "rewards/similarity_reward": 0.6360447704792023,
      "step": 319
    },
    {
      "completion_length": 253.03126525878906,
      "epoch": 0.8533333333333334,
      "grad_norm": 0.6240879358111411,
      "kl": 0.0043487548828125,
      "learning_rate": 2e-06,
      "loss": 0.0052,
      "reward": 0.6519544422626495,
      "reward_std": 0.0803583487868309,
      "rewards/length_reward": 0.0691964440047741,
      "rewards/similarity_reward": 0.5827580094337463,
      "step": 320
    },
    {
      "completion_length": 242.95091247558594,
      "epoch": 0.856,
      "grad_norm": 0.549313108944523,
      "kl": 0.0028839111328125,
      "learning_rate": 2e-06,
      "loss": 0.0208,
      "reward": 0.7446077167987823,
      "reward_std": 0.09269357472658157,
      "rewards/length_reward": 0.07410714775323868,
      "rewards/similarity_reward": 0.6705006062984467,
      "step": 321
    },
    {
      "completion_length": 224.89286041259766,
      "epoch": 0.8586666666666667,
      "grad_norm": 0.7316053385982602,
      "kl": 0.00386810302734375,
      "learning_rate": 2e-06,
      "loss": -0.0086,
      "reward": 0.6810529232025146,
      "reward_std": 0.06378122419118881,
      "rewards/length_reward": 0.071428582072258,
      "rewards/similarity_reward": 0.6096242964267731,
      "step": 322
    },
    {
      "completion_length": 236.2009048461914,
      "epoch": 0.8613333333333333,
      "grad_norm": 0.6760501533209452,
      "kl": 0.0053558349609375,
      "learning_rate": 2e-06,
      "loss": 0.0282,
      "reward": 0.6928089559078217,
      "reward_std": 0.07963042706251144,
      "rewards/length_reward": 0.07410715147852898,
      "rewards/similarity_reward": 0.6187017858028412,
      "step": 323
    },
    {
      "completion_length": 241.27233123779297,
      "epoch": 0.864,
      "grad_norm": 0.7174681952375248,
      "kl": 0.0048065185546875,
      "learning_rate": 2e-06,
      "loss": 0.0184,
      "reward": 0.7107245922088623,
      "reward_std": 0.07196836359798908,
      "rewards/length_reward": 0.07321429252624512,
      "rewards/similarity_reward": 0.6375102698802948,
      "step": 324
    },
    {
      "completion_length": 219.32144165039062,
      "epoch": 0.8666666666666667,
      "grad_norm": 0.6619044506684808,
      "kl": 0.005645751953125,
      "learning_rate": 2e-06,
      "loss": 0.0104,
      "reward": 0.6078022122383118,
      "reward_std": 0.09693693742156029,
      "rewards/length_reward": 0.07098215073347092,
      "rewards/similarity_reward": 0.5368200242519379,
      "step": 325
    },
    {
      "completion_length": 228.84375762939453,
      "epoch": 0.8693333333333333,
      "grad_norm": 0.6998337254787387,
      "kl": 0.00328826904296875,
      "learning_rate": 2e-06,
      "loss": -0.0045,
      "reward": 0.7553087770938873,
      "reward_std": 0.07146935909986496,
      "rewards/length_reward": 0.0736607275903225,
      "rewards/similarity_reward": 0.6816481053829193,
      "step": 326
    },
    {
      "completion_length": 288.2901916503906,
      "epoch": 0.872,
      "grad_norm": 0.6156275180911798,
      "kl": 0.00383758544921875,
      "learning_rate": 2e-06,
      "loss": 0.0138,
      "reward": 0.7228290438652039,
      "reward_std": 0.08385154232382774,
      "rewards/length_reward": 0.071428582072258,
      "rewards/similarity_reward": 0.6514004170894623,
      "step": 327
    },
    {
      "completion_length": 260.9821472167969,
      "epoch": 0.8746666666666667,
      "grad_norm": 0.6463359471511301,
      "kl": 0.0041351318359375,
      "learning_rate": 2e-06,
      "loss": 0.0624,
      "reward": 0.7054339051246643,
      "reward_std": 0.08808758854866028,
      "rewards/length_reward": 0.06339286640286446,
      "rewards/similarity_reward": 0.6420409977436066,
      "step": 328
    },
    {
      "completion_length": 247.34376525878906,
      "epoch": 0.8773333333333333,
      "grad_norm": 0.7642501038807545,
      "kl": 0.00417327880859375,
      "learning_rate": 2e-06,
      "loss": 0.0269,
      "reward": 0.6869321167469025,
      "reward_std": 0.06861887872219086,
      "rewards/length_reward": 0.06741072982549667,
      "rewards/similarity_reward": 0.61952143907547,
      "step": 329
    },
    {
      "completion_length": 267.3928756713867,
      "epoch": 0.88,
      "grad_norm": 0.6571448003835118,
      "kl": 0.0063934326171875,
      "learning_rate": 2e-06,
      "loss": 0.0592,
      "reward": 0.6967031061649323,
      "reward_std": 0.08689341694116592,
      "rewards/length_reward": 0.07455357909202576,
      "rewards/similarity_reward": 0.6221494972705841,
      "step": 330
    },
    {
      "completion_length": 251.84822845458984,
      "epoch": 0.8826666666666667,
      "grad_norm": 0.6160126783055467,
      "kl": 0.00362396240234375,
      "learning_rate": 2e-06,
      "loss": 0.0047,
      "reward": 0.6543110907077789,
      "reward_std": 0.08520649373531342,
      "rewards/length_reward": 0.06562501192092896,
      "rewards/similarity_reward": 0.58868607878685,
      "step": 331
    },
    {
      "completion_length": 277.6384048461914,
      "epoch": 0.8853333333333333,
      "grad_norm": 0.6951712149387916,
      "kl": 0.00399017333984375,
      "learning_rate": 2e-06,
      "loss": -0.0197,
      "reward": 0.6851289868354797,
      "reward_std": 0.09784185141324997,
      "rewards/length_reward": 0.07232143357396126,
      "rewards/similarity_reward": 0.6128075420856476,
      "step": 332
    },
    {
      "completion_length": 256.7366180419922,
      "epoch": 0.888,
      "grad_norm": 0.5992862799755034,
      "kl": 0.004791259765625,
      "learning_rate": 2e-06,
      "loss": 0.0491,
      "reward": 0.6802008152008057,
      "reward_std": 0.09179921820759773,
      "rewards/length_reward": 0.07008929178118706,
      "rewards/similarity_reward": 0.6101114749908447,
      "step": 333
    },
    {
      "completion_length": 215.21430206298828,
      "epoch": 0.8906666666666667,
      "grad_norm": 0.709856632058597,
      "kl": 0.0052032470703125,
      "learning_rate": 2e-06,
      "loss": 0.0124,
      "reward": 0.692722350358963,
      "reward_std": 0.08042297512292862,
      "rewards/length_reward": 0.060714298859238625,
      "rewards/similarity_reward": 0.6320080757141113,
      "step": 334
    },
    {
      "completion_length": 244.75001525878906,
      "epoch": 0.8933333333333333,
      "grad_norm": 0.623193882721459,
      "kl": 0.00424957275390625,
      "learning_rate": 2e-06,
      "loss": 0.0358,
      "reward": 0.7010281980037689,
      "reward_std": 0.08354394137859344,
      "rewards/length_reward": 0.07098215445876122,
      "rewards/similarity_reward": 0.630046010017395,
      "step": 335
    },
    {
      "completion_length": 222.87500762939453,
      "epoch": 0.896,
      "grad_norm": 0.7264124980613719,
      "kl": 0.0045928955078125,
      "learning_rate": 2e-06,
      "loss": 0.0194,
      "reward": 0.6454309821128845,
      "reward_std": 0.10383091494441032,
      "rewards/length_reward": 0.06562501192092896,
      "rewards/similarity_reward": 0.579805999994278,
      "step": 336
    },
    {
      "completion_length": 260.1696548461914,
      "epoch": 0.8986666666666666,
      "grad_norm": 0.5907324344548502,
      "kl": 0.0032806396484375,
      "learning_rate": 2e-06,
      "loss": 0.0162,
      "reward": 0.6906337440013885,
      "reward_std": 0.06872132420539856,
      "rewards/length_reward": 0.07723214477300644,
      "rewards/similarity_reward": 0.6134015619754791,
      "step": 337
    },
    {
      "completion_length": 245.22322845458984,
      "epoch": 0.9013333333333333,
      "grad_norm": 0.6126450019945038,
      "kl": 0.00359344482421875,
      "learning_rate": 2e-06,
      "loss": 0.0172,
      "reward": 0.7013856172561646,
      "reward_std": 0.07948705554008484,
      "rewards/length_reward": 0.07678572088479996,
      "rewards/similarity_reward": 0.6245998740196228,
      "step": 338
    },
    {
      "completion_length": 250.85269927978516,
      "epoch": 0.904,
      "grad_norm": 0.7156180383403373,
      "kl": 0.00537109375,
      "learning_rate": 2e-06,
      "loss": 0.0313,
      "reward": 0.6530185341835022,
      "reward_std": 0.0950470082461834,
      "rewards/length_reward": 0.07187500596046448,
      "rewards/similarity_reward": 0.5811434835195541,
      "step": 339
    },
    {
      "completion_length": 253.91964721679688,
      "epoch": 0.9066666666666666,
      "grad_norm": 0.5742388325200097,
      "kl": 0.004302978515625,
      "learning_rate": 2e-06,
      "loss": 0.01,
      "reward": 0.6900386214256287,
      "reward_std": 0.06002875231206417,
      "rewards/length_reward": 0.07991071790456772,
      "rewards/similarity_reward": 0.6101278960704803,
      "step": 340
    },
    {
      "completion_length": 214.0446548461914,
      "epoch": 0.9093333333333333,
      "grad_norm": 0.7992075593440748,
      "kl": 0.0047760009765625,
      "learning_rate": 2e-06,
      "loss": 0.017,
      "reward": 0.6177321374416351,
      "reward_std": 0.0978124737739563,
      "rewards/length_reward": 0.058482153341174126,
      "rewards/similarity_reward": 0.559249997138977,
      "step": 341
    },
    {
      "completion_length": 244.4732208251953,
      "epoch": 0.912,
      "grad_norm": 0.8258653848140411,
      "kl": 0.0045623779296875,
      "learning_rate": 2e-06,
      "loss": 0.0631,
      "reward": 0.6902224123477936,
      "reward_std": 0.08696568384766579,
      "rewards/length_reward": 0.061607154086232185,
      "rewards/similarity_reward": 0.6286152601242065,
      "step": 342
    },
    {
      "completion_length": 217.93304443359375,
      "epoch": 0.9146666666666666,
      "grad_norm": 0.6997396550994213,
      "kl": 0.005523681640625,
      "learning_rate": 2e-06,
      "loss": 0.0019,
      "reward": 0.6602669954299927,
      "reward_std": 0.09806296601891518,
      "rewards/length_reward": 0.06428572349250317,
      "rewards/similarity_reward": 0.5959812104701996,
      "step": 343
    },
    {
      "completion_length": 285.45538330078125,
      "epoch": 0.9173333333333333,
      "grad_norm": 0.6020909063855457,
      "kl": 0.00421142578125,
      "learning_rate": 2e-06,
      "loss": 0.0088,
      "reward": 0.6885746419429779,
      "reward_std": 0.09227034822106361,
      "rewards/length_reward": 0.07232143729925156,
      "rewards/similarity_reward": 0.6162531673908234,
      "step": 344
    },
    {
      "completion_length": 260.15179443359375,
      "epoch": 0.92,
      "grad_norm": 0.8278574941445127,
      "kl": 0.0059051513671875,
      "learning_rate": 2e-06,
      "loss": 0.1217,
      "reward": 0.6081913709640503,
      "reward_std": 0.18488339334726334,
      "rewards/length_reward": 0.058482155203819275,
      "rewards/similarity_reward": 0.5497092008590698,
      "step": 345
    },
    {
      "completion_length": 272.45091247558594,
      "epoch": 0.9226666666666666,
      "grad_norm": 0.5703333877662986,
      "kl": 0.00386810302734375,
      "learning_rate": 2e-06,
      "loss": 0.021,
      "reward": 0.7158257365226746,
      "reward_std": 0.08523573726415634,
      "rewards/length_reward": 0.07991071790456772,
      "rewards/similarity_reward": 0.6359150111675262,
      "step": 346
    },
    {
      "completion_length": 291.5714416503906,
      "epoch": 0.9253333333333333,
      "grad_norm": 0.664603610390973,
      "kl": 0.00350189208984375,
      "learning_rate": 2e-06,
      "loss": 0.0368,
      "reward": 0.7160574495792389,
      "reward_std": 0.0684958528727293,
      "rewards/length_reward": 0.06651787087321281,
      "rewards/similarity_reward": 0.6495395302772522,
      "step": 347
    },
    {
      "completion_length": 301.0357360839844,
      "epoch": 0.928,
      "grad_norm": 0.5288343523218547,
      "kl": 0.00455474853515625,
      "learning_rate": 2e-06,
      "loss": 0.015,
      "reward": 0.7117947041988373,
      "reward_std": 0.10228880494832993,
      "rewards/length_reward": 0.06741072237491608,
      "rewards/similarity_reward": 0.6443840265274048,
      "step": 348
    },
    {
      "completion_length": 260.44197845458984,
      "epoch": 0.9306666666666666,
      "grad_norm": 0.6372539241149925,
      "kl": 0.00432586669921875,
      "learning_rate": 2e-06,
      "loss": 0.031,
      "reward": 0.7437091171741486,
      "reward_std": 0.07142951153218746,
      "rewards/length_reward": 0.0691964402794838,
      "rewards/similarity_reward": 0.6745127141475677,
      "step": 349
    },
    {
      "completion_length": 266.7232208251953,
      "epoch": 0.9333333333333333,
      "grad_norm": 0.7417544957869086,
      "kl": 0.003936767578125,
      "learning_rate": 2e-06,
      "loss": 0.0914,
      "reward": 0.7335431277751923,
      "reward_std": 0.10938547924160957,
      "rewards/length_reward": 0.07321429252624512,
      "rewards/similarity_reward": 0.6603288650512695,
      "step": 350
    },
    {
      "completion_length": 263.49554443359375,
      "epoch": 0.936,
      "grad_norm": 0.7076854337089484,
      "kl": 0.008270263671875,
      "learning_rate": 2e-06,
      "loss": -0.0148,
      "reward": 0.6161931157112122,
      "reward_std": 0.12820692360401154,
      "rewards/length_reward": 0.06339287385344505,
      "rewards/similarity_reward": 0.5528002381324768,
      "step": 351
    },
    {
      "completion_length": 263.9866180419922,
      "epoch": 0.9386666666666666,
      "grad_norm": 0.5813263791328529,
      "kl": 0.00344085693359375,
      "learning_rate": 2e-06,
      "loss": 0.0071,
      "reward": 0.6747315526008606,
      "reward_std": 0.09237341955304146,
      "rewards/length_reward": 0.06607143953442574,
      "rewards/similarity_reward": 0.6086601614952087,
      "step": 352
    },
    {
      "completion_length": 242.46875762939453,
      "epoch": 0.9413333333333334,
      "grad_norm": 0.6091815171465548,
      "kl": 0.0047607421875,
      "learning_rate": 2e-06,
      "loss": 0.0248,
      "reward": 0.6233691573143005,
      "reward_std": 0.12349110096693039,
      "rewards/length_reward": 0.0647321529686451,
      "rewards/similarity_reward": 0.5586370527744293,
      "step": 353
    },
    {
      "completion_length": 265.0089416503906,
      "epoch": 0.944,
      "grad_norm": 0.5824155704971612,
      "kl": 0.00321197509765625,
      "learning_rate": 2e-06,
      "loss": 0.0139,
      "reward": 0.7225279808044434,
      "reward_std": 0.0762885119765997,
      "rewards/length_reward": 0.0714285857975483,
      "rewards/similarity_reward": 0.651099443435669,
      "step": 354
    },
    {
      "completion_length": 276.56697845458984,
      "epoch": 0.9466666666666667,
      "grad_norm": 0.5410198221272389,
      "kl": 0.0042724609375,
      "learning_rate": 2e-06,
      "loss": -0.0064,
      "reward": 0.7190838754177094,
      "reward_std": 0.11137175559997559,
      "rewards/length_reward": 0.07544643431901932,
      "rewards/similarity_reward": 0.6436374485492706,
      "step": 355
    },
    {
      "completion_length": 299.20982360839844,
      "epoch": 0.9493333333333334,
      "grad_norm": 0.5828251751367955,
      "kl": 0.0042877197265625,
      "learning_rate": 2e-06,
      "loss": 0.0076,
      "reward": 0.7062688171863556,
      "reward_std": 0.0902559757232666,
      "rewards/length_reward": 0.0781250037252903,
      "rewards/similarity_reward": 0.6281438171863556,
      "step": 356
    },
    {
      "completion_length": 261.5714416503906,
      "epoch": 0.952,
      "grad_norm": 0.662401613193247,
      "kl": 0.00417327880859375,
      "learning_rate": 2e-06,
      "loss": 0.1072,
      "reward": 0.6910728216171265,
      "reward_std": 0.12025736272335052,
      "rewards/length_reward": 0.06517858058214188,
      "rewards/similarity_reward": 0.6258941888809204,
      "step": 357
    },
    {
      "completion_length": 273.22322845458984,
      "epoch": 0.9546666666666667,
      "grad_norm": 0.5897487229990825,
      "kl": 0.0035400390625,
      "learning_rate": 2e-06,
      "loss": 0.0006,
      "reward": 0.7075876295566559,
      "reward_std": 0.06886249966919422,
      "rewards/length_reward": 0.07098215445876122,
      "rewards/similarity_reward": 0.6366054713726044,
      "step": 358
    },
    {
      "completion_length": 287.7634048461914,
      "epoch": 0.9573333333333334,
      "grad_norm": 0.5282572684933031,
      "kl": 0.00276947021484375,
      "learning_rate": 2e-06,
      "loss": -0.0016,
      "reward": 0.7663991749286652,
      "reward_std": 0.08505138382315636,
      "rewards/length_reward": 0.07767857611179352,
      "rewards/similarity_reward": 0.6887206435203552,
      "step": 359
    },
    {
      "completion_length": 245.16519165039062,
      "epoch": 0.96,
      "grad_norm": 0.6891670466809943,
      "kl": 0.00555419921875,
      "learning_rate": 2e-06,
      "loss": -0.0045,
      "reward": 0.6559944748878479,
      "reward_std": 0.10601956769824028,
      "rewards/length_reward": 0.0691964402794838,
      "rewards/similarity_reward": 0.5867980420589447,
      "step": 360
    },
    {
      "completion_length": 270.0357208251953,
      "epoch": 0.9626666666666667,
      "grad_norm": 0.5428726789809684,
      "kl": 0.0041961669921875,
      "learning_rate": 2e-06,
      "loss": 0.0106,
      "reward": 0.6817942559719086,
      "reward_std": 0.07338645495474339,
      "rewards/length_reward": 0.07767857611179352,
      "rewards/similarity_reward": 0.6041156351566315,
      "step": 361
    },
    {
      "completion_length": 243.84376525878906,
      "epoch": 0.9653333333333334,
      "grad_norm": 0.643839801785035,
      "kl": 0.0121307373046875,
      "learning_rate": 2e-06,
      "loss": 0.0103,
      "reward": 0.6772570908069611,
      "reward_std": 0.06569460779428482,
      "rewards/length_reward": 0.06964286789298058,
      "rewards/similarity_reward": 0.6076142191886902,
      "step": 362
    },
    {
      "completion_length": 242.50894165039062,
      "epoch": 0.968,
      "grad_norm": 0.5946163970209724,
      "kl": 0.00385284423828125,
      "learning_rate": 2e-06,
      "loss": 0.0095,
      "reward": 0.684212863445282,
      "reward_std": 0.09431854262948036,
      "rewards/length_reward": 0.07455357909202576,
      "rewards/similarity_reward": 0.6096592247486115,
      "step": 363
    },
    {
      "completion_length": 266.8080520629883,
      "epoch": 0.9706666666666667,
      "grad_norm": 0.5952393619644073,
      "kl": 0.0045318603515625,
      "learning_rate": 2e-06,
      "loss": 0.0054,
      "reward": 0.6641288101673126,
      "reward_std": 0.08274504169821739,
      "rewards/length_reward": 0.06651787459850311,
      "rewards/similarity_reward": 0.5976109206676483,
      "step": 364
    },
    {
      "completion_length": 280.81251525878906,
      "epoch": 0.9733333333333334,
      "grad_norm": 0.704408476901472,
      "kl": 0.006378173828125,
      "learning_rate": 2e-06,
      "loss": 0.0302,
      "reward": 0.6689063608646393,
      "reward_std": 0.06241089664399624,
      "rewards/length_reward": 0.06964286789298058,
      "rewards/similarity_reward": 0.5992635488510132,
      "step": 365
    },
    {
      "completion_length": 272.53126525878906,
      "epoch": 0.976,
      "grad_norm": 0.6740721279945435,
      "kl": 0.005615234375,
      "learning_rate": 2e-06,
      "loss": 0.0723,
      "reward": 0.6874203383922577,
      "reward_std": 0.08594712242484093,
      "rewards/length_reward": 0.06785715371370316,
      "rewards/similarity_reward": 0.6195632219314575,
      "step": 366
    },
    {
      "completion_length": 237.32144165039062,
      "epoch": 0.9786666666666667,
      "grad_norm": 0.6453259649723537,
      "kl": 0.0041046142578125,
      "learning_rate": 2e-06,
      "loss": 0.0057,
      "reward": 0.6572641432285309,
      "reward_std": 0.07327758520841599,
      "rewards/length_reward": 0.07053572125732899,
      "rewards/similarity_reward": 0.5867283642292023,
      "step": 367
    },
    {
      "completion_length": 230.68304443359375,
      "epoch": 0.9813333333333333,
      "grad_norm": 0.6672094875832669,
      "kl": 0.00567626953125,
      "learning_rate": 2e-06,
      "loss": -0.0058,
      "reward": 0.6544754207134247,
      "reward_std": 0.09319397434592247,
      "rewards/length_reward": 0.071428582072258,
      "rewards/similarity_reward": 0.5830467939376831,
      "step": 368
    },
    {
      "completion_length": 183.78125762939453,
      "epoch": 0.984,
      "grad_norm": 0.7366023492253149,
      "kl": 0.00495147705078125,
      "learning_rate": 2e-06,
      "loss": -0.0059,
      "reward": 0.6258625388145447,
      "reward_std": 0.08410684764385223,
      "rewards/length_reward": 0.07187500968575478,
      "rewards/similarity_reward": 0.5539875626564026,
      "step": 369
    },
    {
      "completion_length": 242.49555206298828,
      "epoch": 0.9866666666666667,
      "grad_norm": 0.6621715728429993,
      "kl": 0.0055999755859375,
      "learning_rate": 2e-06,
      "loss": 0.0227,
      "reward": 0.6772402822971344,
      "reward_std": 0.08659335412085056,
      "rewards/length_reward": 0.0714285783469677,
      "rewards/similarity_reward": 0.6058117151260376,
      "step": 370
    },
    {
      "completion_length": 223.2366180419922,
      "epoch": 0.9893333333333333,
      "grad_norm": 0.7315442476679772,
      "kl": 0.0076751708984375,
      "learning_rate": 2e-06,
      "loss": 0.0034,
      "reward": 0.6550423204898834,
      "reward_std": 0.08679429814219475,
      "rewards/length_reward": 0.06830358132719994,
      "rewards/similarity_reward": 0.5867387652397156,
      "step": 371
    },
    {
      "completion_length": 279.18751525878906,
      "epoch": 0.992,
      "grad_norm": 0.7043801053601064,
      "kl": 0.005035400390625,
      "learning_rate": 2e-06,
      "loss": 0.0732,
      "reward": 0.6848048269748688,
      "reward_std": 0.10013966262340546,
      "rewards/length_reward": 0.07991071417927742,
      "rewards/similarity_reward": 0.6048941314220428,
      "step": 372
    },
    {
      "completion_length": 221.90625762939453,
      "epoch": 0.9946666666666667,
      "grad_norm": 0.6692794616147932,
      "kl": 0.0042877197265625,
      "learning_rate": 2e-06,
      "loss": -0.0054,
      "reward": 0.6771494746208191,
      "reward_std": 0.0797378458082676,
      "rewards/length_reward": 0.061607155948877335,
      "rewards/similarity_reward": 0.6155422627925873,
      "step": 373
    },
    {
      "completion_length": 287.4330596923828,
      "epoch": 0.9973333333333333,
      "grad_norm": 0.6988535931211539,
      "kl": 0.0048828125,
      "learning_rate": 2e-06,
      "loss": 0.0135,
      "reward": 0.7209034264087677,
      "reward_std": 0.0916130281984806,
      "rewards/length_reward": 0.07321429252624512,
      "rewards/similarity_reward": 0.647689163684845,
      "step": 374
    },
    {
      "completion_length": 234.6508026123047,
      "epoch": 1.0,
      "grad_norm": 0.7155440816719996,
      "kl": 0.0072784423828125,
      "learning_rate": 2e-06,
      "loss": 0.0237,
      "reward": 0.6349575817584991,
      "reward_std": 0.13314566388726234,
      "rewards/length_reward": 0.056696439161896706,
      "rewards/similarity_reward": 0.578261137008667,
      "step": 375
    }
  ],
  "logging_steps": 1,
  "max_steps": 375,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 16,
  "trial_name": null,
  "trial_params": null
}