{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.617372812039915,
  "eval_steps": 500,
  "global_step": 2000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 5.000000000000001e-07,
      "logits/chosen": -1.9473017454147339,
      "logits/rejected": -1.9154374599456787,
      "logps/chosen": -178.9344940185547,
      "logps/rejected": -157.74179077148438,
      "loss": 0.6983,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0024076940026134253,
      "rewards/margins": -0.008672237396240234,
      "rewards/rejected": 0.011079930700361729,
      "step": 1
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.0000000000000002e-06,
      "logits/chosen": -1.6428608894348145,
      "logits/rejected": -1.720033884048462,
      "logps/chosen": -144.77987670898438,
      "logps/rejected": -144.8594207763672,
      "loss": 0.6891,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010233569890260696,
      "rewards/margins": 0.00934591330587864,
      "rewards/rejected": 0.0008876564679667354,
      "step": 2
    },
    {
      "epoch": 0.0,
      "learning_rate": 1.5e-06,
      "logits/chosen": -2.0388426780700684,
      "logits/rejected": -1.988499402999878,
      "logps/chosen": -212.5189208984375,
      "logps/rejected": -199.8199462890625,
      "loss": 0.6912,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.010333014652132988,
      "rewards/margins": 0.005366659723222256,
      "rewards/rejected": 0.004966353997588158,
      "step": 3
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.0000000000000003e-06,
      "logits/chosen": -1.9381685256958008,
      "logits/rejected": -1.9115777015686035,
      "logps/chosen": -141.91700744628906,
      "logps/rejected": -141.35037231445312,
      "loss": 0.6876,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.006317043211311102,
      "rewards/margins": 0.01223981473594904,
      "rewards/rejected": -0.005922770127654076,
      "step": 4
    },
    {
      "epoch": 0.01,
      "learning_rate": 2.5e-06,
      "logits/chosen": -2.0682923793792725,
      "logits/rejected": -2.116490364074707,
      "logps/chosen": -144.48883056640625,
      "logps/rejected": -136.4881591796875,
      "loss": 0.6905,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004738474264740944,
      "rewards/margins": 0.006069636438041925,
      "rewards/rejected": -0.0013311614748090506,
      "step": 5
    },
    {
      "epoch": 0.01,
      "learning_rate": 3e-06,
      "logits/chosen": -1.8117401599884033,
      "logits/rejected": -1.763953447341919,
      "logps/chosen": -151.67367553710938,
      "logps/rejected": -137.02761840820312,
      "loss": 0.6984,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.004828071687370539,
      "rewards/margins": -0.008975815027952194,
      "rewards/rejected": 0.00414774427190423,
      "step": 6
    },
    {
      "epoch": 0.01,
      "learning_rate": 3.5000000000000004e-06,
      "logits/chosen": -2.0915303230285645,
      "logits/rejected": -2.1096673011779785,
      "logps/chosen": -146.31625366210938,
      "logps/rejected": -154.5668487548828,
      "loss": 0.6955,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.002810073085129261,
      "rewards/margins": -0.004074001684784889,
      "rewards/rejected": 0.001263928133994341,
      "step": 7
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.000000000000001e-06,
      "logits/chosen": -2.126337766647339,
      "logits/rejected": -2.126678228378296,
      "logps/chosen": -204.9049072265625,
      "logps/rejected": -203.91268920898438,
      "loss": 0.6962,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008610963821411133,
      "rewards/margins": -0.005686474964022636,
      "rewards/rejected": 0.014297439716756344,
      "step": 8
    },
    {
      "epoch": 0.01,
      "learning_rate": 4.5e-06,
      "logits/chosen": -2.2768051624298096,
      "logits/rejected": -2.281789541244507,
      "logps/chosen": -153.13116455078125,
      "logps/rejected": -150.92642211914062,
      "loss": 0.6874,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01126334723085165,
      "rewards/margins": 0.012444520369172096,
      "rewards/rejected": -0.0011811736039817333,
      "step": 9
    },
    {
      "epoch": 0.01,
      "learning_rate": 5e-06,
      "logits/chosen": -1.9441958665847778,
      "logits/rejected": -1.964548110961914,
      "logps/chosen": -149.23391723632812,
      "logps/rejected": -137.14862060546875,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020777558907866478,
      "rewards/margins": 0.011875724419951439,
      "rewards/rejected": 0.008901833556592464,
      "step": 10
    },
    {
      "epoch": 0.01,
      "learning_rate": 5.500000000000001e-06,
      "logits/chosen": -2.147075653076172,
      "logits/rejected": -2.1675617694854736,
      "logps/chosen": -174.22434997558594,
      "logps/rejected": -178.1639404296875,
      "loss": 0.6947,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.013791371136903763,
      "rewards/margins": -0.0025409706868231297,
      "rewards/rejected": 0.016332341358065605,
      "step": 11
    },
    {
      "epoch": 0.02,
      "learning_rate": 6e-06,
      "logits/chosen": -1.9246153831481934,
      "logits/rejected": -1.9109158515930176,
      "logps/chosen": -150.50784301757812,
      "logps/rejected": -180.81753540039062,
      "loss": 0.7001,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.012552834115922451,
      "rewards/margins": -0.012693023309111595,
      "rewards/rejected": 0.00014019012451171875,
      "step": 12
    },
    {
      "epoch": 0.02,
      "learning_rate": 6.5000000000000004e-06,
      "logits/chosen": -1.9389597177505493,
      "logits/rejected": -1.9608503580093384,
      "logps/chosen": -205.6465606689453,
      "logps/rejected": -201.75946044921875,
      "loss": 0.6865,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.014348938129842281,
      "rewards/margins": 0.013899493962526321,
      "rewards/rejected": 0.00044944253750145435,
      "step": 13
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.000000000000001e-06,
      "logits/chosen": -1.9243704080581665,
      "logits/rejected": -1.9085144996643066,
      "logps/chosen": -201.07188415527344,
      "logps/rejected": -221.47982788085938,
      "loss": 0.6869,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0006500715389847755,
      "rewards/margins": 0.013952446170151234,
      "rewards/rejected": -0.01460251584649086,
      "step": 14
    },
    {
      "epoch": 0.02,
      "learning_rate": 7.5e-06,
      "logits/chosen": -2.1744699478149414,
      "logits/rejected": -2.13222599029541,
      "logps/chosen": -204.4735107421875,
      "logps/rejected": -200.4049072265625,
      "loss": 0.6807,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.017023086547851562,
      "rewards/margins": 0.02616300620138645,
      "rewards/rejected": -0.009139918722212315,
      "step": 15
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.000000000000001e-06,
      "logits/chosen": -2.1755266189575195,
      "logits/rejected": -2.150144338607788,
      "logps/chosen": -160.6802215576172,
      "logps/rejected": -157.8507080078125,
      "loss": 0.6891,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007808923255652189,
      "rewards/margins": 0.009649563580751419,
      "rewards/rejected": -0.0018406407907605171,
      "step": 16
    },
    {
      "epoch": 0.02,
      "learning_rate": 8.500000000000002e-06,
      "logits/chosen": -2.1014106273651123,
      "logits/rejected": -2.065537452697754,
      "logps/chosen": -137.4163360595703,
      "logps/rejected": -137.72653198242188,
      "loss": 0.6789,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.019019413739442825,
      "rewards/margins": 0.030721498653292656,
      "rewards/rejected": -0.01170208491384983,
      "step": 17
    },
    {
      "epoch": 0.02,
      "learning_rate": 9e-06,
      "logits/chosen": -2.126314401626587,
      "logits/rejected": -2.1292574405670166,
      "logps/chosen": -157.85987854003906,
      "logps/rejected": -175.0936279296875,
      "loss": 0.69,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.008974838070571423,
      "rewards/margins": 0.007031917572021484,
      "rewards/rejected": 0.0019429202657192945,
      "step": 18
    },
    {
      "epoch": 0.02,
      "learning_rate": 9.5e-06,
      "logits/chosen": -1.821614384651184,
      "logits/rejected": -1.8386234045028687,
      "logps/chosen": -157.74339294433594,
      "logps/rejected": -173.5108642578125,
      "loss": 0.6854,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0003479006700217724,
      "rewards/margins": 0.016933869570493698,
      "rewards/rejected": -0.016585970297455788,
      "step": 19
    },
    {
      "epoch": 0.03,
      "learning_rate": 1e-05,
      "logits/chosen": -2.3475935459136963,
      "logits/rejected": -2.3527560234069824,
      "logps/chosen": -198.3360137939453,
      "logps/rejected": -197.68064880371094,
      "loss": 0.7084,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.014697913080453873,
      "rewards/margins": -0.029072880744934082,
      "rewards/rejected": 0.014374972321093082,
      "step": 20
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.05e-05,
      "logits/chosen": -1.9768266677856445,
      "logits/rejected": -1.979588508605957,
      "logps/chosen": -138.13516235351562,
      "logps/rejected": -137.23497009277344,
      "loss": 0.7008,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.008711148053407669,
      "rewards/margins": -0.014900517649948597,
      "rewards/rejected": 0.006189371459186077,
      "step": 21
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.1000000000000001e-05,
      "logits/chosen": -2.020932197570801,
      "logits/rejected": -2.0543789863586426,
      "logps/chosen": -143.21487426757812,
      "logps/rejected": -145.47467041015625,
      "loss": 0.7039,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0006606103852391243,
      "rewards/margins": -0.019345475360751152,
      "rewards/rejected": 0.0200060848146677,
      "step": 22
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.1500000000000002e-05,
      "logits/chosen": -1.9338953495025635,
      "logits/rejected": -1.8950414657592773,
      "logps/chosen": -151.0517578125,
      "logps/rejected": -144.9283447265625,
      "loss": 0.6729,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0207887664437294,
      "rewards/margins": 0.0420105941593647,
      "rewards/rejected": -0.02122182957828045,
      "step": 23
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.2e-05,
      "logits/chosen": -2.1229336261749268,
      "logits/rejected": -2.1787612438201904,
      "logps/chosen": -143.4667510986328,
      "logps/rejected": -151.78887939453125,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0076406970620155334,
      "rewards/margins": -0.001663590781390667,
      "rewards/rejected": -0.0059771062806248665,
      "step": 24
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.25e-05,
      "logits/chosen": -1.9385018348693848,
      "logits/rejected": -1.8521825075149536,
      "logps/chosen": -136.98797607421875,
      "logps/rejected": -123.29915618896484,
      "loss": 0.6874,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.016515588387846947,
      "rewards/margins": 0.013585926033556461,
      "rewards/rejected": 0.0029296651482582092,
      "step": 25
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.3000000000000001e-05,
      "logits/chosen": -2.0494840145111084,
      "logits/rejected": -2.0260462760925293,
      "logps/chosen": -206.23294067382812,
      "logps/rejected": -204.59170532226562,
      "loss": 0.7052,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.005298185162246227,
      "rewards/margins": -0.022758912295103073,
      "rewards/rejected": 0.01746072620153427,
      "step": 26
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.3500000000000001e-05,
      "logits/chosen": -2.185777425765991,
      "logits/rejected": -2.16872501373291,
      "logps/chosen": -166.58811950683594,
      "logps/rejected": -168.48207092285156,
      "loss": 0.6994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02519994042813778,
      "rewards/margins": -0.011203411035239697,
      "rewards/rejected": -0.013996529392898083,
      "step": 27
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.4000000000000001e-05,
      "logits/chosen": -2.1363608837127686,
      "logits/rejected": -2.1508541107177734,
      "logps/chosen": -174.48770141601562,
      "logps/rejected": -169.1255645751953,
      "loss": 0.7026,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.008833074010908604,
      "rewards/margins": -0.01709108054637909,
      "rewards/rejected": 0.008258008398115635,
      "step": 28
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.45e-05,
      "logits/chosen": -2.1649487018585205,
      "logits/rejected": -2.170478582382202,
      "logps/chosen": -166.38059997558594,
      "logps/rejected": -170.26541137695312,
      "loss": 0.7105,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.028479862958192825,
      "rewards/margins": -0.0317809097468853,
      "rewards/rejected": 0.0033010481856763363,
      "step": 29
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.5e-05,
      "logits/chosen": -2.190495014190674,
      "logits/rejected": -2.2020809650421143,
      "logps/chosen": -167.83895874023438,
      "logps/rejected": -171.6207275390625,
      "loss": 0.6924,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.007887793704867363,
      "rewards/margins": 0.0026388168334960938,
      "rewards/rejected": 0.005248976871371269,
      "step": 30
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.55e-05,
      "logits/chosen": -2.0097718238830566,
      "logits/rejected": -1.9927585124969482,
      "logps/chosen": -130.2247314453125,
      "logps/rejected": -149.3783416748047,
      "loss": 0.6896,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.010196828283369541,
      "rewards/margins": 0.008201027289032936,
      "rewards/rejected": 0.0019958019256591797,
      "step": 31
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.6000000000000003e-05,
      "logits/chosen": -1.9905472993850708,
      "logits/rejected": -2.0040249824523926,
      "logps/chosen": -176.70541381835938,
      "logps/rejected": -200.47962951660156,
      "loss": 0.7047,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.0031967416871339083,
      "rewards/margins": -0.021031878888607025,
      "rewards/rejected": 0.024228623136878014,
      "step": 32
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.65e-05,
      "logits/chosen": -2.0395946502685547,
      "logits/rejected": -2.019467353820801,
      "logps/chosen": -136.9028778076172,
      "logps/rejected": -118.68156433105469,
      "loss": 0.6872,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.02108776569366455,
      "rewards/margins": 0.013590503484010696,
      "rewards/rejected": 0.007497262209653854,
      "step": 33
    },
    {
      "epoch": 0.04,
      "learning_rate": 1.7000000000000003e-05,
      "logits/chosen": -2.0771450996398926,
      "logits/rejected": -2.1549808979034424,
      "logps/chosen": -133.3074493408203,
      "logps/rejected": -141.6013946533203,
      "loss": 0.6851,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01747903786599636,
      "rewards/margins": 0.017702102661132812,
      "rewards/rejected": -0.00022306526079773903,
      "step": 34
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.75e-05,
      "logits/chosen": -2.1276822090148926,
      "logits/rejected": -2.078389883041382,
      "logps/chosen": -154.18087768554688,
      "logps/rejected": -135.57997131347656,
      "loss": 0.7035,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.009335566312074661,
      "rewards/margins": -0.019415616989135742,
      "rewards/rejected": 0.028751183301210403,
      "step": 35
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.8e-05,
      "logits/chosen": -2.0852906703948975,
      "logits/rejected": -2.04500675201416,
      "logps/chosen": -167.43182373046875,
      "logps/rejected": -164.1446533203125,
      "loss": 0.6959,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02079620398581028,
      "rewards/margins": -0.0038356767036020756,
      "rewards/rejected": 0.024631882086396217,
      "step": 36
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.85e-05,
      "logits/chosen": -1.9053428173065186,
      "logits/rejected": -1.8860125541687012,
      "logps/chosen": -186.79791259765625,
      "logps/rejected": -183.456298828125,
      "loss": 0.7025,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009571265429258347,
      "rewards/margins": -0.01751232147216797,
      "rewards/rejected": 0.027083586901426315,
      "step": 37
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9e-05,
      "logits/chosen": -1.9760335683822632,
      "logits/rejected": -2.062265634536743,
      "logps/chosen": -131.81793212890625,
      "logps/rejected": -126.02755737304688,
      "loss": 0.6897,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.006855441257357597,
      "rewards/margins": 0.009298227727413177,
      "rewards/rejected": -0.016153670847415924,
      "step": 38
    },
    {
      "epoch": 0.05,
      "learning_rate": 1.9500000000000003e-05,
      "logits/chosen": -1.9339022636413574,
      "logits/rejected": -1.9163322448730469,
      "logps/chosen": -154.1663818359375,
      "logps/rejected": -155.90472412109375,
      "loss": 0.6856,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.028609374538064003,
      "rewards/margins": 0.01638217084109783,
      "rewards/rejected": 0.012227201834321022,
      "step": 39
    },
    {
      "epoch": 0.05,
      "learning_rate": 2e-05,
      "logits/chosen": -1.9966247081756592,
      "logits/rejected": -1.949063777923584,
      "logps/chosen": -167.77857971191406,
      "logps/rejected": -169.52732849121094,
      "loss": 0.6966,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09298010170459747,
      "rewards/margins": -0.005213452503085136,
      "rewards/rejected": 0.09819354861974716,
      "step": 40
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.05e-05,
      "logits/chosen": -2.129456043243408,
      "logits/rejected": -2.1342248916625977,
      "logps/chosen": -133.3363494873047,
      "logps/rejected": -137.60580444335938,
      "loss": 0.6937,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.060414716601371765,
      "rewards/margins": 0.00036644982174038887,
      "rewards/rejected": 0.060048267245292664,
      "step": 41
    },
    {
      "epoch": 0.05,
      "learning_rate": 2.1e-05,
      "logits/chosen": -2.113492012023926,
      "logits/rejected": -2.149386167526245,
      "logps/chosen": -158.78790283203125,
      "logps/rejected": -151.22894287109375,
      "loss": 0.6959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.042578794062137604,
      "rewards/margins": -0.003336430061608553,
      "rewards/rejected": 0.04591522365808487,
      "step": 42
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.15e-05,
      "logits/chosen": -2.14461088180542,
      "logits/rejected": -2.127142906188965,
      "logps/chosen": -161.65231323242188,
      "logps/rejected": -193.1698760986328,
      "loss": 0.7047,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.04428081959486008,
      "rewards/margins": -0.021442033350467682,
      "rewards/rejected": 0.06572284549474716,
      "step": 43
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.2000000000000003e-05,
      "logits/chosen": -1.9628074169158936,
      "logits/rejected": -1.9140665531158447,
      "logps/chosen": -165.8412628173828,
      "logps/rejected": -159.8250274658203,
      "loss": 0.699,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01591024361550808,
      "rewards/margins": -0.009422186762094498,
      "rewards/rejected": -0.006488058716058731,
      "step": 44
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.25e-05,
      "logits/chosen": -2.009124517440796,
      "logits/rejected": -2.051636219024658,
      "logps/chosen": -157.3372039794922,
      "logps/rejected": -168.30613708496094,
      "loss": 0.6646,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06606419384479523,
      "rewards/margins": 0.060197923332452774,
      "rewards/rejected": 0.0058662667870521545,
      "step": 45
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.3000000000000003e-05,
      "logits/chosen": -1.9960308074951172,
      "logits/rejected": -2.013731002807617,
      "logps/chosen": -152.45262145996094,
      "logps/rejected": -152.74990844726562,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.041900113224983215,
      "rewards/margins": 0.005311486776918173,
      "rewards/rejected": 0.036588624119758606,
      "step": 46
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.35e-05,
      "logits/chosen": -2.084881544113159,
      "logits/rejected": -2.051604747772217,
      "logps/chosen": -147.22457885742188,
      "logps/rejected": -130.90673828125,
      "loss": 0.7003,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.027727343142032623,
      "rewards/margins": -0.011356806382536888,
      "rewards/rejected": 0.03908415138721466,
      "step": 47
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.4e-05,
      "logits/chosen": -2.172280788421631,
      "logits/rejected": -2.161738872528076,
      "logps/chosen": -169.7745819091797,
      "logps/rejected": -169.87936401367188,
      "loss": 0.6954,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.026907825842499733,
      "rewards/margins": -0.0014351843856275082,
      "rewards/rejected": 0.02834300883114338,
      "step": 48
    },
    {
      "epoch": 0.06,
      "learning_rate": 2.45e-05,
      "logits/chosen": -2.127919912338257,
      "logits/rejected": -2.0433998107910156,
      "logps/chosen": -170.56634521484375,
      "logps/rejected": -158.96981811523438,
      "loss": 0.7064,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.04663754254579544,
      "rewards/margins": -0.023569582030177116,
      "rewards/rejected": 0.07020711898803711,
      "step": 49
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.5e-05,
      "logits/chosen": -2.0349974632263184,
      "logits/rejected": -1.9954936504364014,
      "logps/chosen": -198.20065307617188,
      "logps/rejected": -184.9116973876953,
      "loss": 0.6746,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.031090188771486282,
      "rewards/margins": 0.039133429527282715,
      "rewards/rejected": -0.008043241687119007,
      "step": 50
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.5500000000000003e-05,
      "logits/chosen": -2.1735117435455322,
      "logits/rejected": -2.1993043422698975,
      "logps/chosen": -152.16355895996094,
      "logps/rejected": -152.68006896972656,
      "loss": 0.7076,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.03591465950012207,
      "rewards/margins": -0.024077631533145905,
      "rewards/rejected": 0.05999229848384857,
      "step": 51
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.6000000000000002e-05,
      "logits/chosen": -1.999128818511963,
      "logits/rejected": -2.059635877609253,
      "logps/chosen": -201.5443878173828,
      "logps/rejected": -227.005859375,
      "loss": 0.6924,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02559967152774334,
      "rewards/margins": 0.004609701223671436,
      "rewards/rejected": 0.02098996937274933,
      "step": 52
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.6500000000000004e-05,
      "logits/chosen": -2.162651538848877,
      "logits/rejected": -2.1881916522979736,
      "logps/chosen": -124.34934997558594,
      "logps/rejected": -130.11549377441406,
      "loss": 0.6881,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08756237477064133,
      "rewards/margins": 0.012067937292158604,
      "rewards/rejected": 0.0754944384098053,
      "step": 53
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.7000000000000002e-05,
      "logits/chosen": -1.9938299655914307,
      "logits/rejected": -2.034883499145508,
      "logps/chosen": -167.63168334960938,
      "logps/rejected": -171.09104919433594,
      "loss": 0.6914,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0470796562731266,
      "rewards/margins": 0.00963954720646143,
      "rewards/rejected": 0.03744010999798775,
      "step": 54
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.7500000000000004e-05,
      "logits/chosen": -2.0790414810180664,
      "logits/rejected": -2.1538212299346924,
      "logps/chosen": -155.3251953125,
      "logps/rejected": -174.05441284179688,
      "loss": 0.701,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.041025254875421524,
      "rewards/margins": -0.013429548591375351,
      "rewards/rejected": 0.054454803466796875,
      "step": 55
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.8000000000000003e-05,
      "logits/chosen": -2.1940407752990723,
      "logits/rejected": -2.2308764457702637,
      "logps/chosen": -187.58314514160156,
      "logps/rejected": -175.3198699951172,
      "loss": 0.663,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.09865675121545792,
      "rewards/margins": 0.06462635844945908,
      "rewards/rejected": 0.03403039276599884,
      "step": 56
    },
    {
      "epoch": 0.07,
      "learning_rate": 2.8499999999999998e-05,
      "logits/chosen": -2.0591015815734863,
      "logits/rejected": -2.084754705429077,
      "logps/chosen": -170.54522705078125,
      "logps/rejected": -157.01759338378906,
      "loss": 0.6995,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.030951734632253647,
      "rewards/margins": -0.004660461097955704,
      "rewards/rejected": 0.03561220318078995,
      "step": 57
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.9e-05,
      "logits/chosen": -1.9737329483032227,
      "logits/rejected": -1.965449333190918,
      "logps/chosen": -149.2818603515625,
      "logps/rejected": -140.09564208984375,
      "loss": 0.6789,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07900600135326385,
      "rewards/margins": 0.030826283618807793,
      "rewards/rejected": 0.04817971587181091,
      "step": 58
    },
    {
      "epoch": 0.08,
      "learning_rate": 2.95e-05,
      "logits/chosen": -1.9386465549468994,
      "logits/rejected": -1.9863896369934082,
      "logps/chosen": -148.0929718017578,
      "logps/rejected": -152.614990234375,
      "loss": 0.6867,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.06615930050611496,
      "rewards/margins": 0.014939931221306324,
      "rewards/rejected": 0.05121936649084091,
      "step": 59
    },
    {
      "epoch": 0.08,
      "learning_rate": 3e-05,
      "logits/chosen": -2.1406030654907227,
      "logits/rejected": -2.140972852706909,
      "logps/chosen": -164.55569458007812,
      "logps/rejected": -160.45802307128906,
      "loss": 0.6848,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.07306110858917236,
      "rewards/margins": 0.02018454112112522,
      "rewards/rejected": 0.05287656933069229,
      "step": 60
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.05e-05,
      "logits/chosen": -2.1821627616882324,
      "logits/rejected": -2.23848819732666,
      "logps/chosen": -154.3368682861328,
      "logps/rejected": -153.51547241210938,
      "loss": 0.6941,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.07242944836616516,
      "rewards/margins": 9.055249392986298e-05,
      "rewards/rejected": 0.07233888655900955,
      "step": 61
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.1e-05,
      "logits/chosen": -2.1834821701049805,
      "logits/rejected": -2.132404327392578,
      "logps/chosen": -148.86456298828125,
      "logps/rejected": -149.2091522216797,
      "loss": 0.7058,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.09177599102258682,
      "rewards/margins": -0.021642468869686127,
      "rewards/rejected": 0.11341846734285355,
      "step": 62
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.15e-05,
      "logits/chosen": -2.037621259689331,
      "logits/rejected": -2.0467352867126465,
      "logps/chosen": -151.49147033691406,
      "logps/rejected": -165.01246643066406,
      "loss": 0.7098,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.1084907054901123,
      "rewards/margins": -0.023577161133289337,
      "rewards/rejected": 0.13206787407398224,
      "step": 63
    },
    {
      "epoch": 0.08,
      "learning_rate": 3.2000000000000005e-05,
      "logits/chosen": -2.062051773071289,
      "logits/rejected": -1.9661951065063477,
      "logps/chosen": -139.70193481445312,
      "logps/rejected": -155.814697265625,
      "loss": 0.6917,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0038987197913229465,
      "rewards/margins": 0.0076565514318645,
      "rewards/rejected": -0.0037578358314931393,
      "step": 64
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.2500000000000004e-05,
      "logits/chosen": -2.132296562194824,
      "logits/rejected": -2.091379404067993,
      "logps/chosen": -148.2380828857422,
      "logps/rejected": -155.97186279296875,
      "loss": 0.6824,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.061218734830617905,
      "rewards/margins": 0.02413906902074814,
      "rewards/rejected": 0.037079669535160065,
      "step": 65
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.3e-05,
      "logits/chosen": -1.8095799684524536,
      "logits/rejected": -1.8316328525543213,
      "logps/chosen": -168.23243713378906,
      "logps/rejected": -163.02633666992188,
      "loss": 0.7047,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.09200361371040344,
      "rewards/margins": -0.017515014857053757,
      "rewards/rejected": 0.1095186173915863,
      "step": 66
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.35e-05,
      "logits/chosen": -1.8896549940109253,
      "logits/rejected": -1.9064245223999023,
      "logps/chosen": -178.73973083496094,
      "logps/rejected": -180.1619873046875,
      "loss": 0.7115,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.024672793224453926,
      "rewards/margins": -0.03270921856164932,
      "rewards/rejected": 0.0573820136487484,
      "step": 67
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.4000000000000007e-05,
      "logits/chosen": -1.7941502332687378,
      "logits/rejected": -1.8412476778030396,
      "logps/chosen": -134.13589477539062,
      "logps/rejected": -135.673828125,
      "loss": 0.6453,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.13328862190246582,
      "rewards/margins": 0.10187779366970062,
      "rewards/rejected": 0.03141083940863609,
      "step": 68
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.45e-05,
      "logits/chosen": -2.1372315883636475,
      "logits/rejected": -2.158930778503418,
      "logps/chosen": -142.1901397705078,
      "logps/rejected": -140.1024169921875,
      "loss": 0.7298,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.04432840272784233,
      "rewards/margins": -0.06238814443349838,
      "rewards/rejected": 0.10671653598546982,
      "step": 69
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.5e-05,
      "logits/chosen": -2.2218027114868164,
      "logits/rejected": -2.2194578647613525,
      "logps/chosen": -172.63040161132812,
      "logps/rejected": -171.96815490722656,
      "loss": 0.7005,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.033330656588077545,
      "rewards/margins": -0.012100504711270332,
      "rewards/rejected": 0.04543116316199303,
      "step": 70
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.55e-05,
      "logits/chosen": -2.012197971343994,
      "logits/rejected": -2.0255186557769775,
      "logps/chosen": -217.9222869873047,
      "logps/rejected": -226.47398376464844,
      "loss": 0.6984,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.058617155998945236,
      "rewards/margins": -0.005790230818092823,
      "rewards/rejected": 0.06440739333629608,
      "step": 71
    },
    {
      "epoch": 0.09,
      "learning_rate": 3.6e-05,
      "logits/chosen": -2.14078688621521,
      "logits/rejected": -2.111306667327881,
      "logps/chosen": -160.2899169921875,
      "logps/rejected": -154.1366424560547,
      "loss": 0.6881,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.074866384267807,
      "rewards/margins": 0.01636476442217827,
      "rewards/rejected": 0.05850161984562874,
      "step": 72
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.65e-05,
      "logits/chosen": -1.9476792812347412,
      "logits/rejected": -1.9206452369689941,
      "logps/chosen": -156.6016845703125,
      "logps/rejected": -155.9908905029297,
      "loss": 0.6852,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.004470587708055973,
      "rewards/margins": 0.024280693382024765,
      "rewards/rejected": -0.019810102880001068,
      "step": 73
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.7e-05,
      "logits/chosen": -2.07848858833313,
      "logits/rejected": -2.1027286052703857,
      "logps/chosen": -153.7880096435547,
      "logps/rejected": -151.38841247558594,
      "loss": 0.7136,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.049511026591062546,
      "rewards/margins": -0.0270721185952425,
      "rewards/rejected": 0.0765831470489502,
      "step": 74
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.7500000000000003e-05,
      "logits/chosen": -2.0560519695281982,
      "logits/rejected": -2.0237274169921875,
      "logps/chosen": -168.32672119140625,
      "logps/rejected": -166.328125,
      "loss": 0.6885,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01440761424601078,
      "rewards/margins": 0.0161895789206028,
      "rewards/rejected": -0.0017819646745920181,
      "step": 75
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.8e-05,
      "logits/chosen": -1.9260778427124023,
      "logits/rejected": -1.9472447633743286,
      "logps/chosen": -145.28759765625,
      "logps/rejected": -149.98504638671875,
      "loss": 0.701,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.034238480031490326,
      "rewards/margins": -0.005065919831395149,
      "rewards/rejected": 0.039304401725530624,
      "step": 76
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.85e-05,
      "logits/chosen": -1.6790459156036377,
      "logits/rejected": -1.6938380002975464,
      "logps/chosen": -169.58250427246094,
      "logps/rejected": -171.97097778320312,
      "loss": 0.7036,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12939399480819702,
      "rewards/margins": -0.008996442891657352,
      "rewards/rejected": 0.1383904367685318,
      "step": 77
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.9000000000000006e-05,
      "logits/chosen": -2.2361156940460205,
      "logits/rejected": -2.1365509033203125,
      "logps/chosen": -207.76535034179688,
      "logps/rejected": -197.6790313720703,
      "loss": 0.6763,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.10514166951179504,
      "rewards/margins": 0.03811817243695259,
      "rewards/rejected": 0.06702349334955215,
      "step": 78
    },
    {
      "epoch": 0.1,
      "learning_rate": 3.9500000000000005e-05,
      "logits/chosen": -2.0133023262023926,
      "logits/rejected": -2.0518369674682617,
      "logps/chosen": -143.17681884765625,
      "logps/rejected": -136.48175048828125,
      "loss": 0.6976,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05453691631555557,
      "rewards/margins": -0.002915834542363882,
      "rewards/rejected": 0.05745274946093559,
      "step": 79
    },
    {
      "epoch": 0.1,
      "learning_rate": 4e-05,
      "logits/chosen": -2.024761438369751,
      "logits/rejected": -2.054973840713501,
      "logps/chosen": -153.51858520507812,
      "logps/rejected": -142.02432250976562,
      "loss": 0.6967,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.01629617251455784,
      "rewards/margins": -0.0009380597621202469,
      "rewards/rejected": 0.017234232276678085,
      "step": 80
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.05e-05,
      "logits/chosen": -2.0025839805603027,
      "logits/rejected": -2.0151402950286865,
      "logps/chosen": -159.1929931640625,
      "logps/rejected": -168.1624755859375,
      "loss": 0.687,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.06722717732191086,
      "rewards/margins": 0.03626187518239021,
      "rewards/rejected": 0.030965294688940048,
      "step": 81
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.1e-05,
      "logits/chosen": -1.9327080249786377,
      "logits/rejected": -1.947874665260315,
      "logps/chosen": -215.408935546875,
      "logps/rejected": -199.43902587890625,
      "loss": 0.7457,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.1100463718175888,
      "rewards/margins": -0.09321331977844238,
      "rewards/rejected": -0.016833044588565826,
      "step": 82
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.15e-05,
      "logits/chosen": -1.9191949367523193,
      "logits/rejected": -1.9307305812835693,
      "logps/chosen": -170.3133544921875,
      "logps/rejected": -154.84254455566406,
      "loss": 0.7279,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03949739784002304,
      "rewards/margins": -0.06344561278820038,
      "rewards/rejected": 0.023948216810822487,
      "step": 83
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.2e-05,
      "logits/chosen": -1.8360068798065186,
      "logits/rejected": -1.8425829410552979,
      "logps/chosen": -165.39141845703125,
      "logps/rejected": -161.62539672851562,
      "loss": 0.7216,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07253064215183258,
      "rewards/margins": -0.03998390585184097,
      "rewards/rejected": 0.11251455545425415,
      "step": 84
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.25e-05,
      "logits/chosen": -1.9030508995056152,
      "logits/rejected": -1.9330065250396729,
      "logps/chosen": -209.78103637695312,
      "logps/rejected": -220.98043823242188,
      "loss": 0.6309,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0014497279189527035,
      "rewards/margins": 0.13583050668239594,
      "rewards/rejected": -0.1343807578086853,
      "step": 85
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.3e-05,
      "logits/chosen": -1.9742060899734497,
      "logits/rejected": -1.9866974353790283,
      "logps/chosen": -171.7286376953125,
      "logps/rejected": -168.8912811279297,
      "loss": 0.7484,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.025174710899591446,
      "rewards/margins": -0.09477367997169495,
      "rewards/rejected": 0.1199483871459961,
      "step": 86
    },
    {
      "epoch": 0.11,
      "learning_rate": 4.35e-05,
      "logits/chosen": -2.1695969104766846,
      "logits/rejected": -2.252450704574585,
      "logps/chosen": -125.7203598022461,
      "logps/rejected": -158.59054565429688,
      "loss": 0.7255,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.010992627590894699,
      "rewards/margins": -0.052416570484638214,
      "rewards/rejected": 0.04142393916845322,
      "step": 87
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.4000000000000006e-05,
      "logits/chosen": -1.9488649368286133,
      "logits/rejected": -1.9779549837112427,
      "logps/chosen": -182.26705932617188,
      "logps/rejected": -173.61148071289062,
      "loss": 0.7215,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.07614026218652725,
      "rewards/margins": -0.042962007224559784,
      "rewards/rejected": 0.11910226941108704,
      "step": 88
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.4500000000000004e-05,
      "logits/chosen": -1.9029545783996582,
      "logits/rejected": -1.9423730373382568,
      "logps/chosen": -200.17457580566406,
      "logps/rejected": -173.20709228515625,
      "loss": 0.733,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.07196169346570969,
      "rewards/margins": -0.0681467056274414,
      "rewards/rejected": 0.1401083916425705,
      "step": 89
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.5e-05,
      "logits/chosen": -2.3139305114746094,
      "logits/rejected": -2.351571798324585,
      "logps/chosen": -146.32611083984375,
      "logps/rejected": -156.42579650878906,
      "loss": 0.7095,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0665954127907753,
      "rewards/margins": -0.01309509202837944,
      "rewards/rejected": -0.05350032076239586,
      "step": 90
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.55e-05,
      "logits/chosen": -1.9338881969451904,
      "logits/rejected": -1.9347970485687256,
      "logps/chosen": -143.15924072265625,
      "logps/rejected": -152.71804809570312,
      "loss": 0.7129,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.010173199698328972,
      "rewards/margins": -0.03304898738861084,
      "rewards/rejected": 0.022875778377056122,
      "step": 91
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.600000000000001e-05,
      "logits/chosen": -1.7374138832092285,
      "logits/rejected": -1.7747021913528442,
      "logps/chosen": -187.83636474609375,
      "logps/rejected": -160.97560119628906,
      "loss": 0.682,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06837701797485352,
      "rewards/margins": 0.042838774621486664,
      "rewards/rejected": 0.02553824707865715,
      "step": 92
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.6500000000000005e-05,
      "logits/chosen": -2.21986722946167,
      "logits/rejected": -2.2679433822631836,
      "logps/chosen": -147.17849731445312,
      "logps/rejected": -152.95431518554688,
      "loss": 0.6438,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08736428618431091,
      "rewards/margins": 0.10906204581260681,
      "rewards/rejected": -0.021697763353586197,
      "step": 93
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.7e-05,
      "logits/chosen": -2.187103271484375,
      "logits/rejected": -2.239633083343506,
      "logps/chosen": -124.42755126953125,
      "logps/rejected": -125.82307434082031,
      "loss": 0.6312,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03520822525024414,
      "rewards/margins": 0.14293043315410614,
      "rewards/rejected": -0.1077222228050232,
      "step": 94
    },
    {
      "epoch": 0.12,
      "learning_rate": 4.75e-05,
      "logits/chosen": -2.1484622955322266,
      "logits/rejected": -2.167619228363037,
      "logps/chosen": -139.31021118164062,
      "logps/rejected": -148.8551483154297,
      "loss": 0.6474,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11308951675891876,
      "rewards/margins": 0.10877098888158798,
      "rewards/rejected": 0.004318520426750183,
      "step": 95
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.8e-05,
      "logits/chosen": -2.1004369258880615,
      "logits/rejected": -2.0368094444274902,
      "logps/chosen": -142.34259033203125,
      "logps/rejected": -131.41265869140625,
      "loss": 0.6511,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06739182770252228,
      "rewards/margins": 0.0982145369052887,
      "rewards/rejected": -0.03082270734012127,
      "step": 96
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.85e-05,
      "logits/chosen": -2.269822597503662,
      "logits/rejected": -2.3615293502807617,
      "logps/chosen": -159.7416534423828,
      "logps/rejected": -158.27255249023438,
      "loss": 0.7067,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.042380258440971375,
      "rewards/margins": -0.016107436269521713,
      "rewards/rejected": -0.02627282217144966,
      "step": 97
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9e-05,
      "logits/chosen": -2.1764469146728516,
      "logits/rejected": -2.1454200744628906,
      "logps/chosen": -142.2655029296875,
      "logps/rejected": -153.2463836669922,
      "loss": 0.7423,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.0025094300508499146,
      "rewards/margins": -0.08575483411550522,
      "rewards/rejected": 0.08324538916349411,
      "step": 98
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9500000000000004e-05,
      "logits/chosen": -2.193749189376831,
      "logits/rejected": -2.1985344886779785,
      "logps/chosen": -187.168701171875,
      "logps/rejected": -197.92352294921875,
      "loss": 0.6663,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06072089821100235,
      "rewards/margins": 0.07174615561962128,
      "rewards/rejected": -0.011025259271264076,
      "step": 99
    },
    {
      "epoch": 0.13,
      "learning_rate": 5e-05,
      "logits/chosen": -2.068103551864624,
      "logits/rejected": -2.0740597248077393,
      "logps/chosen": -144.1133270263672,
      "logps/rejected": -148.28744506835938,
      "loss": 0.7095,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04808714985847473,
      "rewards/margins": -0.004095175303518772,
      "rewards/rejected": 0.05218231678009033,
      "step": 100
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.999997432392803e-05,
      "logits/chosen": -1.8877215385437012,
      "logits/rejected": -1.8843252658843994,
      "logps/chosen": -212.08737182617188,
      "logps/rejected": -159.77037048339844,
      "loss": 0.6442,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.05051098391413689,
      "rewards/margins": 0.11454086750745773,
      "rewards/rejected": -0.06402988731861115,
      "step": 101
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9999897295764844e-05,
      "logits/chosen": -2.040144205093384,
      "logits/rejected": -2.0494396686553955,
      "logps/chosen": -160.35443115234375,
      "logps/rejected": -169.1414794921875,
      "loss": 0.8033,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.07908819615840912,
      "rewards/margins": -0.19221995770931244,
      "rewards/rejected": 0.11313176155090332,
      "step": 102
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.9999768915668665e-05,
      "logits/chosen": -2.148070812225342,
      "logits/rejected": -2.1513681411743164,
      "logps/chosen": -145.71995544433594,
      "logps/rejected": -140.0707550048828,
      "loss": 0.7564,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12240414321422577,
      "rewards/margins": -0.10564970970153809,
      "rewards/rejected": -0.01675444096326828,
      "step": 103
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999958918390321e-05,
      "logits/chosen": -2.0417237281799316,
      "logits/rejected": -1.9771349430084229,
      "logps/chosen": -146.751708984375,
      "logps/rejected": -139.89523315429688,
      "loss": 0.7258,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.053674325346946716,
      "rewards/margins": -0.05639982223510742,
      "rewards/rejected": 0.0027255089953541756,
      "step": 104
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999935810083766e-05,
      "logits/chosen": -2.003383159637451,
      "logits/rejected": -1.9574511051177979,
      "logps/chosen": -173.7794647216797,
      "logps/rejected": -171.2220916748047,
      "loss": 0.7666,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.051795199513435364,
      "rewards/margins": -0.1156022921204567,
      "rewards/rejected": 0.06380710750818253,
      "step": 105
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999907566694667e-05,
      "logits/chosen": -2.0609560012817383,
      "logits/rejected": -2.101062536239624,
      "logps/chosen": -143.58428955078125,
      "logps/rejected": -162.1249237060547,
      "loss": 0.7173,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.04189272224903107,
      "rewards/margins": -0.02848353236913681,
      "rewards/rejected": -0.013409186154603958,
      "step": 106
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9998741882810384e-05,
      "logits/chosen": -1.862243890762329,
      "logits/rejected": -1.8240137100219727,
      "logps/chosen": -177.3428192138672,
      "logps/rejected": -179.76573181152344,
      "loss": 0.7098,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03088255040347576,
      "rewards/margins": -0.002100745216012001,
      "rewards/rejected": -0.028781799599528313,
      "step": 107
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999835674911443e-05,
      "logits/chosen": -1.7740706205368042,
      "logits/rejected": -1.810904860496521,
      "logps/chosen": -203.03497314453125,
      "logps/rejected": -209.43789672851562,
      "loss": 0.66,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06684975326061249,
      "rewards/margins": 0.08228196948766708,
      "rewards/rejected": -0.015432218089699745,
      "step": 108
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.999792026664991e-05,
      "logits/chosen": -1.946571707725525,
      "logits/rejected": -1.9524767398834229,
      "logps/chosen": -160.1234130859375,
      "logps/rejected": -150.89892578125,
      "loss": 0.6772,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.01813810132443905,
      "rewards/margins": 0.042000893503427505,
      "rewards/rejected": -0.06013898551464081,
      "step": 109
    },
    {
      "epoch": 0.14,
      "learning_rate": 4.9997432436313384e-05,
      "logits/chosen": -2.24985933303833,
      "logits/rejected": -2.2947096824645996,
      "logps/chosen": -147.19674682617188,
      "logps/rejected": -147.4495086669922,
      "loss": 0.7232,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14067719876766205,
      "rewards/margins": -0.044903866946697235,
      "rewards/rejected": -0.09577332437038422,
      "step": 110
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.99968932591069e-05,
      "logits/chosen": -2.289724826812744,
      "logits/rejected": -2.1994781494140625,
      "logps/chosen": -170.31321716308594,
      "logps/rejected": -153.826904296875,
      "loss": 0.7918,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1584378331899643,
      "rewards/margins": -0.16181893646717072,
      "rewards/rejected": 0.0033811070024967194,
      "step": 111
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999630273613799e-05,
      "logits/chosen": -1.6163581609725952,
      "logits/rejected": -1.6122593879699707,
      "logps/chosen": -168.62997436523438,
      "logps/rejected": -222.83038330078125,
      "loss": 0.6669,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.057224374264478683,
      "rewards/margins": 0.09167467057704926,
      "rewards/rejected": -0.03445029631257057,
      "step": 112
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999566086861961e-05,
      "logits/chosen": -2.1440043449401855,
      "logits/rejected": -2.1370463371276855,
      "logps/chosen": -133.85556030273438,
      "logps/rejected": -146.81748962402344,
      "loss": 0.7124,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06403251737356186,
      "rewards/margins": -0.009389929473400116,
      "rewards/rejected": -0.054642580449581146,
      "step": 113
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999496765787024e-05,
      "logits/chosen": -2.1842727661132812,
      "logits/rejected": -2.2154009342193604,
      "logps/chosen": -151.34469604492188,
      "logps/rejected": -153.56678771972656,
      "loss": 0.6244,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.009856656193733215,
      "rewards/margins": 0.15723924338817596,
      "rewards/rejected": -0.14738260209560394,
      "step": 114
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9994223105313774e-05,
      "logits/chosen": -2.212733745574951,
      "logits/rejected": -2.2484166622161865,
      "logps/chosen": -129.7595672607422,
      "logps/rejected": -129.0322265625,
      "loss": 0.6389,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0931883379817009,
      "rewards/margins": 0.12833790481090546,
      "rewards/rejected": -0.035149574279785156,
      "step": 115
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.9993427212479606e-05,
      "logits/chosen": -2.1278915405273438,
      "logits/rejected": -2.1659746170043945,
      "logps/chosen": -166.4802703857422,
      "logps/rejected": -177.78651428222656,
      "loss": 0.7355,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13353124260902405,
      "rewards/margins": -0.06453972309827805,
      "rewards/rejected": -0.068991519510746,
      "step": 116
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999257998100254e-05,
      "logits/chosen": -2.3552539348602295,
      "logits/rejected": -2.4131429195404053,
      "logps/chosen": -167.33251953125,
      "logps/rejected": -171.97781372070312,
      "loss": 0.7883,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13030581176280975,
      "rewards/margins": -0.1426038295030594,
      "rewards/rejected": 0.012298017740249634,
      "step": 117
    },
    {
      "epoch": 0.15,
      "learning_rate": 4.999168141262289e-05,
      "logits/chosen": -2.211704969406128,
      "logits/rejected": -2.204983711242676,
      "logps/chosen": -206.1199188232422,
      "logps/rejected": -197.96646118164062,
      "loss": 0.7095,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04005058482289314,
      "rewards/margins": -0.022744348272681236,
      "rewards/rejected": -0.017306234687566757,
      "step": 118
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9990731509186376e-05,
      "logits/chosen": -2.1870293617248535,
      "logits/rejected": -2.1281723976135254,
      "logps/chosen": -155.93276977539062,
      "logps/rejected": -145.3558807373047,
      "loss": 0.6718,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.005868477746844292,
      "rewards/margins": 0.055788375437259674,
      "rewards/rejected": -0.049919892102479935,
      "step": 119
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.998973027264419e-05,
      "logits/chosen": -1.9813153743743896,
      "logits/rejected": -1.9183677434921265,
      "logps/chosen": -150.78817749023438,
      "logps/rejected": -147.15357971191406,
      "loss": 0.7039,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.17385157942771912,
      "rewards/margins": -0.001015951856970787,
      "rewards/rejected": -0.1728356033563614,
      "step": 120
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.998867770505295e-05,
      "logits/chosen": -2.391598701477051,
      "logits/rejected": -2.3418595790863037,
      "logps/chosen": -167.2589111328125,
      "logps/rejected": -178.03309631347656,
      "loss": 0.6671,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0277772955596447,
      "rewards/margins": 0.0589999184012413,
      "rewards/rejected": -0.0867772102355957,
      "step": 121
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9987573808574726e-05,
      "logits/chosen": -1.9197032451629639,
      "logits/rejected": -2.0230495929718018,
      "logps/chosen": -114.3215560913086,
      "logps/rejected": -125.48574829101562,
      "loss": 0.6731,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02883918769657612,
      "rewards/margins": 0.04838750883936882,
      "rewards/rejected": -0.0772266834974289,
      "step": 122
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9986418585477016e-05,
      "logits/chosen": -2.1796233654022217,
      "logits/rejected": -2.1268649101257324,
      "logps/chosen": -144.93719482421875,
      "logps/rejected": -125.82449340820312,
      "loss": 0.7403,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0862671434879303,
      "rewards/margins": -0.07632093131542206,
      "rewards/rejected": -0.009946208447217941,
      "step": 123
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.998521203813274e-05,
      "logits/chosen": -1.9971880912780762,
      "logits/rejected": -2.0406086444854736,
      "logps/chosen": -154.99465942382812,
      "logps/rejected": -154.3177947998047,
      "loss": 0.753,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1506948173046112,
      "rewards/margins": -0.05945264920592308,
      "rewards/rejected": -0.09124217927455902,
      "step": 124
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9983954169020256e-05,
      "logits/chosen": -2.0880544185638428,
      "logits/rejected": -2.123401165008545,
      "logps/chosen": -169.77288818359375,
      "logps/rejected": -179.76673889160156,
      "loss": 0.6491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16007837653160095,
      "rewards/margins": 0.12425337731838226,
      "rewards/rejected": -0.284331738948822,
      "step": 125
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.9982644980723334e-05,
      "logits/chosen": -2.232131242752075,
      "logits/rejected": -2.2934720516204834,
      "logps/chosen": -159.07867431640625,
      "logps/rejected": -156.19357299804688,
      "loss": 0.7076,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1424141526222229,
      "rewards/margins": 0.0020025279372930527,
      "rewards/rejected": -0.1444166749715805,
      "step": 126
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.998128447593117e-05,
      "logits/chosen": -2.07320499420166,
      "logits/rejected": -2.0934879779815674,
      "logps/chosen": -173.55859375,
      "logps/rejected": -171.46405029296875,
      "loss": 0.6545,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.15888014435768127,
      "rewards/margins": 0.14077186584472656,
      "rewards/rejected": -0.29965201020240784,
      "step": 127
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.997987265743834e-05,
      "logits/chosen": -2.186350107192993,
      "logits/rejected": -2.153662919998169,
      "logps/chosen": -137.228271484375,
      "logps/rejected": -150.18634033203125,
      "loss": 0.6362,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06281783431768417,
      "rewards/margins": 0.16370470821857452,
      "rewards/rejected": -0.2265225499868393,
      "step": 128
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.997840952814484e-05,
      "logits/chosen": -2.0112736225128174,
      "logits/rejected": -2.0177900791168213,
      "logps/chosen": -194.06292724609375,
      "logps/rejected": -184.00973510742188,
      "loss": 0.7179,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.24062253534793854,
      "rewards/margins": -0.023746546357870102,
      "rewards/rejected": -0.21687600016593933,
      "step": 129
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9976895091056075e-05,
      "logits/chosen": -2.066251277923584,
      "logits/rejected": -2.070246458053589,
      "logps/chosen": -136.72486877441406,
      "logps/rejected": -135.44017028808594,
      "loss": 0.6315,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02474498748779297,
      "rewards/margins": 0.1634208709001541,
      "rewards/rejected": -0.13867586851119995,
      "step": 130
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9975329349282826e-05,
      "logits/chosen": -1.9598355293273926,
      "logits/rejected": -2.0056591033935547,
      "logps/chosen": -123.45001220703125,
      "logps/rejected": -134.2187042236328,
      "loss": 0.6282,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.014127110131084919,
      "rewards/margins": 0.1591949462890625,
      "rewards/rejected": -0.17332205176353455,
      "step": 131
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.9973712306041256e-05,
      "logits/chosen": -2.207669734954834,
      "logits/rejected": -2.261643886566162,
      "logps/chosen": -143.61402893066406,
      "logps/rejected": -150.94285583496094,
      "loss": 0.7032,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08026294410228729,
      "rewards/margins": -0.0010632979683578014,
      "rewards/rejected": -0.0791996419429779,
      "step": 132
    },
    {
      "epoch": 0.17,
      "learning_rate": 4.997204396465292e-05,
      "logits/chosen": -2.2536189556121826,
      "logits/rejected": -2.234713554382324,
      "logps/chosen": -166.07725524902344,
      "logps/rejected": -153.78976440429688,
      "loss": 0.7129,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.194389209151268,
      "rewards/margins": -0.020291997119784355,
      "rewards/rejected": -0.1740972250699997,
      "step": 133
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.997032432854472e-05,
      "logits/chosen": -2.1735360622406006,
      "logits/rejected": -2.2178611755371094,
      "logps/chosen": -196.8380126953125,
      "logps/rejected": -206.2840118408203,
      "loss": 0.7158,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.007880209013819695,
      "rewards/margins": -0.0179959274828434,
      "rewards/rejected": 0.010115718469023705,
      "step": 134
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.996855340124894e-05,
      "logits/chosen": -2.069317579269409,
      "logits/rejected": -2.0932023525238037,
      "logps/chosen": -135.76324462890625,
      "logps/rejected": -132.4375,
      "loss": 0.7887,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.13397419452667236,
      "rewards/margins": -0.15053679049015045,
      "rewards/rejected": 0.016562584787607193,
      "step": 135
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.996673118640323e-05,
      "logits/chosen": -2.2751736640930176,
      "logits/rejected": -2.262275218963623,
      "logps/chosen": -144.22671508789062,
      "logps/rejected": -146.7581787109375,
      "loss": 0.6953,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.08663605898618698,
      "rewards/margins": 0.018158910796046257,
      "rewards/rejected": -0.10479498654603958,
      "step": 136
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.996485768775055e-05,
      "logits/chosen": -2.041444778442383,
      "logits/rejected": -2.091449022293091,
      "logps/chosen": -143.69158935546875,
      "logps/rejected": -157.0668182373047,
      "loss": 0.7353,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.209483340382576,
      "rewards/margins": -0.06017661839723587,
      "rewards/rejected": -0.14930672943592072,
      "step": 137
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.996293290913926e-05,
      "logits/chosen": -1.9326127767562866,
      "logits/rejected": -2.0368032455444336,
      "logps/chosen": -147.84547424316406,
      "logps/rejected": -203.94833374023438,
      "loss": 0.7451,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20097234845161438,
      "rewards/margins": -0.03583552688360214,
      "rewards/rejected": -0.16513679921627045,
      "step": 138
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.9960956854522986e-05,
      "logits/chosen": -2.267336368560791,
      "logits/rejected": -2.2412469387054443,
      "logps/chosen": -134.1835174560547,
      "logps/rejected": -136.10162353515625,
      "loss": 0.7371,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.18377897143363953,
      "rewards/margins": -0.07501955330371857,
      "rewards/rejected": -0.10875942558050156,
      "step": 139
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.995892952796074e-05,
      "logits/chosen": -2.3473498821258545,
      "logits/rejected": -2.2889528274536133,
      "logps/chosen": -179.09548950195312,
      "logps/rejected": -176.6771697998047,
      "loss": 0.6946,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.167083278298378,
      "rewards/margins": 0.044769808650016785,
      "rewards/rejected": -0.21185307204723358,
      "step": 140
    },
    {
      "epoch": 0.18,
      "learning_rate": 4.995685093361682e-05,
      "logits/chosen": -2.046065330505371,
      "logits/rejected": -2.105637311935425,
      "logps/chosen": -156.4433135986328,
      "logps/rejected": -193.69764709472656,
      "loss": 0.6491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.10867753624916077,
      "rewards/margins": 0.12942247092723846,
      "rewards/rejected": -0.23810002207756042,
      "step": 141
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.9954721075760824e-05,
      "logits/chosen": -2.135222911834717,
      "logits/rejected": -2.0551419258117676,
      "logps/chosen": -160.54461669921875,
      "logps/rejected": -154.21876525878906,
      "loss": 0.7472,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3932434618473053,
      "rewards/margins": -0.07500467449426651,
      "rewards/rejected": -0.318238765001297,
      "step": 142
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.995253995876767e-05,
      "logits/chosen": -2.36671781539917,
      "logits/rejected": -2.3601675033569336,
      "logps/chosen": -161.03819274902344,
      "logps/rejected": -143.7089385986328,
      "loss": 0.811,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.37680506706237793,
      "rewards/margins": -0.192378431558609,
      "rewards/rejected": -0.18442663550376892,
      "step": 143
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.995030758711756e-05,
      "logits/chosen": -1.7832953929901123,
      "logits/rejected": -1.7478176355361938,
      "logps/chosen": -143.82127380371094,
      "logps/rejected": -146.1196746826172,
      "loss": 0.7393,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.32953861355781555,
      "rewards/margins": -0.02949490211904049,
      "rewards/rejected": -0.3000437021255493,
      "step": 144
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.994802396539598e-05,
      "logits/chosen": -1.5314689874649048,
      "logits/rejected": -1.5684059858322144,
      "logps/chosen": -239.08958435058594,
      "logps/rejected": -253.18838500976562,
      "loss": 0.6769,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.31847676634788513,
      "rewards/margins": 0.07425765693187714,
      "rewards/rejected": -0.3927344083786011,
      "step": 145
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.994568909829368e-05,
      "logits/chosen": -2.0874621868133545,
      "logits/rejected": -2.074917793273926,
      "logps/chosen": -161.60157775878906,
      "logps/rejected": -152.967529296875,
      "loss": 0.7313,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.36861109733581543,
      "rewards/margins": -0.05294780433177948,
      "rewards/rejected": -0.31566327810287476,
      "step": 146
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.9943302990606684e-05,
      "logits/chosen": -1.996845006942749,
      "logits/rejected": -1.9692479372024536,
      "logps/chosen": -157.77352905273438,
      "logps/rejected": -132.0946044921875,
      "loss": 0.8205,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.32108697295188904,
      "rewards/margins": -0.1798648089170456,
      "rewards/rejected": -0.14122214913368225,
      "step": 147
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.994086564723626e-05,
      "logits/chosen": -2.1677422523498535,
      "logits/rejected": -2.18229341506958,
      "logps/chosen": -160.99119567871094,
      "logps/rejected": -180.38742065429688,
      "loss": 0.6763,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5339494943618774,
      "rewards/margins": 0.05808330327272415,
      "rewards/rejected": -0.592032790184021,
      "step": 148
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.9938377073188905e-05,
      "logits/chosen": -2.201998233795166,
      "logits/rejected": -2.1696887016296387,
      "logps/chosen": -155.02134704589844,
      "logps/rejected": -140.8427734375,
      "loss": 0.7243,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29963669180870056,
      "rewards/margins": -0.020382262766361237,
      "rewards/rejected": -0.2792544364929199,
      "step": 149
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.993583727357638e-05,
      "logits/chosen": -2.0553178787231445,
      "logits/rejected": -2.0267903804779053,
      "logps/chosen": -148.95738220214844,
      "logps/rejected": -158.5399932861328,
      "loss": 0.6162,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2186848223209381,
      "rewards/margins": 0.20554782450199127,
      "rewards/rejected": -0.4242326319217682,
      "step": 150
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.993324625361565e-05,
      "logits/chosen": -2.2440099716186523,
      "logits/rejected": -2.1926894187927246,
      "logps/chosen": -185.78314208984375,
      "logps/rejected": -184.54876708984375,
      "loss": 0.6397,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2549251317977905,
      "rewards/margins": 0.24536924064159393,
      "rewards/rejected": -0.5002943873405457,
      "step": 151
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.993060401862888e-05,
      "logits/chosen": -1.9377892017364502,
      "logits/rejected": -1.967207908630371,
      "logps/chosen": -145.74510192871094,
      "logps/rejected": -142.37782287597656,
      "loss": 0.7606,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3630293905735016,
      "rewards/margins": -0.08116517215967178,
      "rewards/rejected": -0.281864196062088,
      "step": 152
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.9927910574043465e-05,
      "logits/chosen": -2.01324462890625,
      "logits/rejected": -2.029172658920288,
      "logps/chosen": -123.41838836669922,
      "logps/rejected": -119.00106048583984,
      "loss": 0.7249,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.20013180375099182,
      "rewards/margins": -0.028365857899188995,
      "rewards/rejected": -0.17176595330238342,
      "step": 153
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.992516592539196e-05,
      "logits/chosen": -1.8588542938232422,
      "logits/rejected": -1.865120530128479,
      "logps/chosen": -193.2555389404297,
      "logps/rejected": -200.77032470703125,
      "loss": 0.7573,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4952069818973541,
      "rewards/margins": -0.08547386527061462,
      "rewards/rejected": -0.4097330868244171,
      "step": 154
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.9922370078312105e-05,
      "logits/chosen": -2.0865638256073,
      "logits/rejected": -2.0685646533966064,
      "logps/chosen": -156.38755798339844,
      "logps/rejected": -151.69525146484375,
      "loss": 0.7112,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.34599804878234863,
      "rewards/margins": -0.008161775767803192,
      "rewards/rejected": -0.33783626556396484,
      "step": 155
    },
    {
      "epoch": 0.2,
      "learning_rate": 4.991952303854682e-05,
      "logits/chosen": -2.1627612113952637,
      "logits/rejected": -2.19240665435791,
      "logps/chosen": -149.98419189453125,
      "logps/rejected": -151.00982666015625,
      "loss": 0.646,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3006647229194641,
      "rewards/margins": 0.12499865889549255,
      "rewards/rejected": -0.42566338181495667,
      "step": 156
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.9916624811944175e-05,
      "logits/chosen": -1.9495761394500732,
      "logits/rejected": -2.0748660564422607,
      "logps/chosen": -159.08567810058594,
      "logps/rejected": -180.21310424804688,
      "loss": 0.707,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.39853742718696594,
      "rewards/margins": 0.021911904215812683,
      "rewards/rejected": -0.42044931650161743,
      "step": 157
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.991367540445735e-05,
      "logits/chosen": -2.0296339988708496,
      "logits/rejected": -1.9863965511322021,
      "logps/chosen": -164.78256225585938,
      "logps/rejected": -165.06405639648438,
      "loss": 0.6399,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.22124487161636353,
      "rewards/margins": 0.14064420759677887,
      "rewards/rejected": -0.3618890643119812,
      "step": 158
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.991067482214471e-05,
      "logits/chosen": -1.9162077903747559,
      "logits/rejected": -1.9429746866226196,
      "logps/chosen": -177.34408569335938,
      "logps/rejected": -191.58206176757812,
      "loss": 0.6414,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5006837844848633,
      "rewards/margins": 0.1467215120792389,
      "rewards/rejected": -0.6474053263664246,
      "step": 159
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.9907623071169686e-05,
      "logits/chosen": -1.9484643936157227,
      "logits/rejected": -1.9940898418426514,
      "logps/chosen": -135.9336700439453,
      "logps/rejected": -147.29998779296875,
      "loss": 0.6828,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2519877552986145,
      "rewards/margins": 0.03291063383221626,
      "rewards/rejected": -0.28489840030670166,
      "step": 160
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.990452015780085e-05,
      "logits/chosen": -1.9855961799621582,
      "logits/rejected": -2.01076340675354,
      "logps/chosen": -140.24473571777344,
      "logps/rejected": -147.60067749023438,
      "loss": 0.6506,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.20326684415340424,
      "rewards/margins": 0.11952166259288788,
      "rewards/rejected": -0.3227885365486145,
      "step": 161
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.9901366088411846e-05,
      "logits/chosen": -1.8572781085968018,
      "logits/rejected": -1.886858582496643,
      "logps/chosen": -159.31396484375,
      "logps/rejected": -150.73434448242188,
      "loss": 0.6815,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.3573068380355835,
      "rewards/margins": 0.08972961455583572,
      "rewards/rejected": -0.447036474943161,
      "step": 162
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.98981608694814e-05,
      "logits/chosen": -2.1316683292388916,
      "logits/rejected": -2.108705759048462,
      "logps/chosen": -166.07952880859375,
      "logps/rejected": -165.3894805908203,
      "loss": 0.7855,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.4258570075035095,
      "rewards/margins": -0.1352803260087967,
      "rewards/rejected": -0.29057663679122925,
      "step": 163
    },
    {
      "epoch": 0.21,
      "learning_rate": 4.9894904507593316e-05,
      "logits/chosen": -2.1435980796813965,
      "logits/rejected": -2.0952234268188477,
      "logps/chosen": -179.2945098876953,
      "logps/rejected": -177.13780212402344,
      "loss": 0.8496,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.6829274892807007,
      "rewards/margins": -0.2457571029663086,
      "rewards/rejected": -0.4371703267097473,
      "step": 164
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.989159700943643e-05,
      "logits/chosen": -2.087015151977539,
      "logits/rejected": -2.0333518981933594,
      "logps/chosen": -156.10824584960938,
      "logps/rejected": -135.26223754882812,
      "loss": 0.7424,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5498830676078796,
      "rewards/margins": -0.06063465029001236,
      "rewards/rejected": -0.4892484247684479,
      "step": 165
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.988823838180464e-05,
      "logits/chosen": -2.076122999191284,
      "logits/rejected": -1.985759973526001,
      "logps/chosen": -157.4197540283203,
      "logps/rejected": -160.52862548828125,
      "loss": 0.6492,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21956738829612732,
      "rewards/margins": 0.1264350414276123,
      "rewards/rejected": -0.3460024297237396,
      "step": 166
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.988482863159684e-05,
      "logits/chosen": -1.8897199630737305,
      "logits/rejected": -1.9176274538040161,
      "logps/chosen": -185.94778442382812,
      "logps/rejected": -197.8125,
      "loss": 0.7095,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.23862342536449432,
      "rewards/margins": 0.02010211907327175,
      "rewards/rejected": -0.2587255537509918,
      "step": 167
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.988136776581696e-05,
      "logits/chosen": -2.0815958976745605,
      "logits/rejected": -2.0887794494628906,
      "logps/chosen": -173.06820678710938,
      "logps/rejected": -177.52330017089844,
      "loss": 0.6666,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.41561299562454224,
      "rewards/margins": 0.10110354423522949,
      "rewards/rejected": -0.5167165994644165,
      "step": 168
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.9877855791573915e-05,
      "logits/chosen": -1.8423527479171753,
      "logits/rejected": -1.9063888788223267,
      "logps/chosen": -123.87655639648438,
      "logps/rejected": -143.618408203125,
      "loss": 0.659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08650312572717667,
      "rewards/margins": 0.08998794108629227,
      "rewards/rejected": -0.17649102210998535,
      "step": 169
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.9874292716081595e-05,
      "logits/chosen": -1.977996587753296,
      "logits/rejected": -2.031097650527954,
      "logps/chosen": -147.41297912597656,
      "logps/rejected": -145.47198486328125,
      "loss": 0.65,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14451590180397034,
      "rewards/margins": 0.10963588207960129,
      "rewards/rejected": -0.2541517913341522,
      "step": 170
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.9870678546658865e-05,
      "logits/chosen": -1.904013752937317,
      "logits/rejected": -1.9502118825912476,
      "logps/chosen": -149.9442596435547,
      "logps/rejected": -156.0302734375,
      "loss": 0.6812,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.31452393531799316,
      "rewards/margins": 0.06542593240737915,
      "rewards/rejected": -0.3799498677253723,
      "step": 171
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9867013290729535e-05,
      "logits/chosen": -1.8673038482666016,
      "logits/rejected": -1.8840340375900269,
      "logps/chosen": -136.29080200195312,
      "logps/rejected": -140.873046875,
      "loss": 0.7688,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3597078323364258,
      "rewards/margins": -0.09261485189199448,
      "rewards/rejected": -0.2670930027961731,
      "step": 172
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.986329695582237e-05,
      "logits/chosen": -1.965585708618164,
      "logits/rejected": -1.9712939262390137,
      "logps/chosen": -170.4312286376953,
      "logps/rejected": -177.6762237548828,
      "loss": 0.6966,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2586846947669983,
      "rewards/margins": 0.04440504312515259,
      "rewards/rejected": -0.3030897378921509,
      "step": 173
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.985952954957103e-05,
      "logits/chosen": -1.9653695821762085,
      "logits/rejected": -1.9432342052459717,
      "logps/chosen": -145.86233520507812,
      "logps/rejected": -127.70154571533203,
      "loss": 0.7328,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.14063119888305664,
      "rewards/margins": -0.0184502974152565,
      "rewards/rejected": -0.12218090891838074,
      "step": 174
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.985571107971408e-05,
      "logits/chosen": -2.048206090927124,
      "logits/rejected": -2.040398359298706,
      "logps/chosen": -150.42953491210938,
      "logps/rejected": -161.55064392089844,
      "loss": 0.6729,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.23834238946437836,
      "rewards/margins": 0.059960223734378815,
      "rewards/rejected": -0.29830265045166016,
      "step": 175
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9851841554095e-05,
      "logits/chosen": -2.0315334796905518,
      "logits/rejected": -2.008814811706543,
      "logps/chosen": -171.3214569091797,
      "logps/rejected": -145.38833618164062,
      "loss": 0.664,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2100733369588852,
      "rewards/margins": 0.09207681566476822,
      "rewards/rejected": -0.3021501302719116,
      "step": 176
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.9847920980662134e-05,
      "logits/chosen": -2.062809467315674,
      "logits/rejected": -2.053818941116333,
      "logps/chosen": -139.9019775390625,
      "logps/rejected": -137.94232177734375,
      "loss": 0.6947,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2767728269100189,
      "rewards/margins": 0.011704735457897186,
      "rewards/rejected": -0.2884775400161743,
      "step": 177
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.984394936746865e-05,
      "logits/chosen": -2.2114696502685547,
      "logits/rejected": -2.3026535511016846,
      "logps/chosen": -155.966552734375,
      "logps/rejected": -169.798583984375,
      "loss": 0.786,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.33644577860832214,
      "rewards/margins": -0.15074411034584045,
      "rewards/rejected": -0.1857016235589981,
      "step": 178
    },
    {
      "epoch": 0.23,
      "learning_rate": 4.98399267226726e-05,
      "logits/chosen": -2.354398727416992,
      "logits/rejected": -2.3184590339660645,
      "logps/chosen": -175.42050170898438,
      "logps/rejected": -143.98519897460938,
      "loss": 0.7219,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3392341136932373,
      "rewards/margins": -0.028377681970596313,
      "rewards/rejected": -0.3108564615249634,
      "step": 179
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9835853054536846e-05,
      "logits/chosen": -1.9018394947052002,
      "logits/rejected": -1.85094153881073,
      "logps/chosen": -199.07778930664062,
      "logps/rejected": -232.95819091796875,
      "loss": 0.7405,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3375914692878723,
      "rewards/margins": -0.021947531029582024,
      "rewards/rejected": -0.31564390659332275,
      "step": 180
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.9831728371429046e-05,
      "logits/chosen": -2.243464231491089,
      "logits/rejected": -2.2446773052215576,
      "logps/chosen": -133.00210571289062,
      "logps/rejected": -138.46485900878906,
      "loss": 0.7021,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.23482827842235565,
      "rewards/margins": 0.007431086152791977,
      "rewards/rejected": -0.24225935339927673,
      "step": 181
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.982755268182164e-05,
      "logits/chosen": -2.26414155960083,
      "logits/rejected": -2.140368700027466,
      "logps/chosen": -149.4698028564453,
      "logps/rejected": -123.35488891601562,
      "loss": 0.8467,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3118973970413208,
      "rewards/margins": -0.22676469385623932,
      "rewards/rejected": -0.0851326733827591,
      "step": 182
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.982332599429187e-05,
      "logits/chosen": -2.044692039489746,
      "logits/rejected": -1.9895075559616089,
      "logps/chosen": -143.49749755859375,
      "logps/rejected": -170.93624877929688,
      "loss": 0.6813,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05278599262237549,
      "rewards/margins": 0.0477883517742157,
      "rewards/rejected": -0.10057434439659119,
      "step": 183
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.981904831752171e-05,
      "logits/chosen": -2.3891890048980713,
      "logits/rejected": -2.4430036544799805,
      "logps/chosen": -135.30807495117188,
      "logps/rejected": -145.2589874267578,
      "loss": 0.7079,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.05791349709033966,
      "rewards/margins": 0.005677835550159216,
      "rewards/rejected": -0.06359133124351501,
      "step": 184
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.981471966029787e-05,
      "logits/chosen": -2.2154502868652344,
      "logits/rejected": -2.205904006958008,
      "logps/chosen": -185.5220947265625,
      "logps/rejected": -211.70501708984375,
      "loss": 0.7164,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.21913209557533264,
      "rewards/margins": 0.028864163905382156,
      "rewards/rejected": -0.24799621105194092,
      "step": 185
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.981034003151178e-05,
      "logits/chosen": -2.2102999687194824,
      "logits/rejected": -2.261056900024414,
      "logps/chosen": -200.81346130371094,
      "logps/rejected": -215.66226196289062,
      "loss": 0.663,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.18210916221141815,
      "rewards/margins": 0.13367104530334473,
      "rewards/rejected": -0.3157802224159241,
      "step": 186
    },
    {
      "epoch": 0.24,
      "learning_rate": 4.980590944015958e-05,
      "logits/chosen": -2.360344648361206,
      "logits/rejected": -2.3027474880218506,
      "logps/chosen": -161.29354858398438,
      "logps/rejected": -146.9613037109375,
      "loss": 0.7363,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.11364208161830902,
      "rewards/margins": -0.03879079967737198,
      "rewards/rejected": -0.07485126703977585,
      "step": 187
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.98014278953421e-05,
      "logits/chosen": -2.300621747970581,
      "logits/rejected": -2.2820940017700195,
      "logps/chosen": -166.09739685058594,
      "logps/rejected": -163.3979034423828,
      "loss": 0.6378,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.017802832648158073,
      "rewards/margins": 0.2077583521604538,
      "rewards/rejected": -0.18995548784732819,
      "step": 188
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.979689540626479e-05,
      "logits/chosen": -2.056394577026367,
      "logits/rejected": -2.069194793701172,
      "logps/chosen": -138.0771942138672,
      "logps/rejected": -125.778076171875,
      "loss": 0.7441,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.13161520659923553,
      "rewards/margins": -0.06547226756811142,
      "rewards/rejected": -0.06614293903112411,
      "step": 189
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9792311982237774e-05,
      "logits/chosen": -1.7692331075668335,
      "logits/rejected": -1.78682541847229,
      "logps/chosen": -184.9560546875,
      "logps/rejected": -196.3815155029297,
      "loss": 0.7329,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2540504038333893,
      "rewards/margins": -0.049651261419057846,
      "rewards/rejected": -0.20439916849136353,
      "step": 190
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.9787677632675825e-05,
      "logits/chosen": -2.1416497230529785,
      "logits/rejected": -2.1960196495056152,
      "logps/chosen": -161.20358276367188,
      "logps/rejected": -173.38148498535156,
      "loss": 0.7561,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16931605339050293,
      "rewards/margins": -0.0933171734213829,
      "rewards/rejected": -0.07599887996912003,
      "step": 191
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.978299236709826e-05,
      "logits/chosen": -2.149383544921875,
      "logits/rejected": -2.1642093658447266,
      "logps/chosen": -184.3213348388672,
      "logps/rejected": -146.78817749023438,
      "loss": 0.712,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.02038545347750187,
      "rewards/margins": 0.031157677993178368,
      "rewards/rejected": -0.051543138921260834,
      "step": 192
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.977825619512904e-05,
      "logits/chosen": -2.1460795402526855,
      "logits/rejected": -2.201239824295044,
      "logps/chosen": -147.57321166992188,
      "logps/rejected": -139.89291381835938,
      "loss": 0.7989,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.1287820041179657,
      "rewards/margins": -0.1620349884033203,
      "rewards/rejected": 0.033253006637096405,
      "step": 193
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.977346912649666e-05,
      "logits/chosen": -2.237581729888916,
      "logits/rejected": -2.2825090885162354,
      "logps/chosen": -138.7524871826172,
      "logps/rejected": -174.55783081054688,
      "loss": 0.6347,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.032103441655635834,
      "rewards/margins": 0.143732950091362,
      "rewards/rejected": -0.17583641409873962,
      "step": 194
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.9768631171034175e-05,
      "logits/chosen": -2.133234739303589,
      "logits/rejected": -2.1712846755981445,
      "logps/chosen": -168.8503875732422,
      "logps/rejected": -162.818603515625,
      "loss": 0.8217,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11681299656629562,
      "rewards/margins": -0.1929503083229065,
      "rewards/rejected": 0.07613730430603027,
      "step": 195
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.9763742338679145e-05,
      "logits/chosen": -2.1170601844787598,
      "logits/rejected": -2.1072490215301514,
      "logps/chosen": -161.21279907226562,
      "logps/rejected": -178.30638122558594,
      "loss": 0.6846,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.10452104359865189,
      "rewards/margins": 0.06742212921380997,
      "rewards/rejected": -0.17194317281246185,
      "step": 196
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.975880263947367e-05,
      "logits/chosen": -2.262809991836548,
      "logits/rejected": -2.249521017074585,
      "logps/chosen": -142.46505737304688,
      "logps/rejected": -142.214599609375,
      "loss": 0.7602,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10644787549972534,
      "rewards/margins": -0.08289748430252075,
      "rewards/rejected": -0.02355036698281765,
      "step": 197
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.9753812083564304e-05,
      "logits/chosen": -2.270545482635498,
      "logits/rejected": -2.2995519638061523,
      "logps/chosen": -180.8372039794922,
      "logps/rejected": -184.7505340576172,
      "loss": 0.693,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05116385966539383,
      "rewards/margins": 0.05982666835188866,
      "rewards/rejected": -0.008662798441946507,
      "step": 198
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.974877068120208e-05,
      "logits/chosen": -1.9270200729370117,
      "logits/rejected": -1.962209701538086,
      "logps/chosen": -186.33587646484375,
      "logps/rejected": -227.984619140625,
      "loss": 0.7574,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.28816187381744385,
      "rewards/margins": -0.04220404103398323,
      "rewards/rejected": -0.24595780670642853,
      "step": 199
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.974367844274248e-05,
      "logits/chosen": -2.477126121520996,
      "logits/rejected": -2.4897990226745605,
      "logps/chosen": -120.29115295410156,
      "logps/rejected": -115.61780548095703,
      "loss": 0.6899,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.002930285409092903,
      "rewards/margins": 0.039310526102781296,
      "rewards/rejected": -0.04224081337451935,
      "step": 200
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.973853537864538e-05,
      "logits/chosen": -2.2658283710479736,
      "logits/rejected": -2.26222562789917,
      "logps/chosen": -129.32443237304688,
      "logps/rejected": -130.61666870117188,
      "loss": 0.6441,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06606045365333557,
      "rewards/margins": 0.13228707015514374,
      "rewards/rejected": -0.06622661650180817,
      "step": 201
    },
    {
      "epoch": 0.26,
      "learning_rate": 4.973334149947508e-05,
      "logits/chosen": -2.133833408355713,
      "logits/rejected": -2.12565279006958,
      "logps/chosen": -169.78045654296875,
      "logps/rejected": -154.434814453125,
      "loss": 0.695,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.02759992890059948,
      "rewards/margins": 0.031831562519073486,
      "rewards/rejected": -0.0042316243052482605,
      "step": 202
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.972809681590026e-05,
      "logits/chosen": -2.2754271030426025,
      "logits/rejected": -2.2876760959625244,
      "logps/chosen": -195.24818420410156,
      "logps/rejected": -206.7046661376953,
      "loss": 0.7125,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.03773926943540573,
      "rewards/margins": -0.024865150451660156,
      "rewards/rejected": 0.06260443478822708,
      "step": 203
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.972280133869396e-05,
      "logits/chosen": -2.323434829711914,
      "logits/rejected": -2.3305745124816895,
      "logps/chosen": -160.62684631347656,
      "logps/rejected": -162.4305419921875,
      "loss": 0.7052,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10412123799324036,
      "rewards/margins": -0.009756050072610378,
      "rewards/rejected": -0.0943651795387268,
      "step": 204
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.971745507873352e-05,
      "logits/chosen": -2.3126134872436523,
      "logits/rejected": -2.3449103832244873,
      "logps/chosen": -164.61329650878906,
      "logps/rejected": -156.84730529785156,
      "loss": 0.6569,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07602532207965851,
      "rewards/margins": 0.11013013869524002,
      "rewards/rejected": -0.03410482034087181,
      "step": 205
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.971205804700063e-05,
      "logits/chosen": -1.984785556793213,
      "logits/rejected": -2.0369019508361816,
      "logps/chosen": -160.10220336914062,
      "logps/rejected": -179.91224670410156,
      "loss": 0.8494,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1521720588207245,
      "rewards/margins": -0.21498528122901917,
      "rewards/rejected": 0.06281323730945587,
      "step": 206
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.970661025458125e-05,
      "logits/chosen": -2.309006452560425,
      "logits/rejected": -2.3352603912353516,
      "logps/chosen": -161.5703887939453,
      "logps/rejected": -163.7167205810547,
      "loss": 0.6492,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05860462412238121,
      "rewards/margins": 0.11608318984508514,
      "rewards/rejected": -0.05747856944799423,
      "step": 207
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.9701111712665625e-05,
      "logits/chosen": -2.227006673812866,
      "logits/rejected": -2.1848740577697754,
      "logps/chosen": -177.54269409179688,
      "logps/rejected": -171.7179718017578,
      "loss": 0.7767,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.17108993232250214,
      "rewards/margins": -0.11503319442272186,
      "rewards/rejected": -0.05605673789978027,
      "step": 208
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.969556243254822e-05,
      "logits/chosen": -2.226109743118286,
      "logits/rejected": -2.2309632301330566,
      "logps/chosen": -126.68124389648438,
      "logps/rejected": -135.05209350585938,
      "loss": 0.6762,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.12173338234424591,
      "rewards/margins": 0.05880265310406685,
      "rewards/rejected": 0.06293072551488876,
      "step": 209
    },
    {
      "epoch": 0.27,
      "learning_rate": 4.968996242562774e-05,
      "logits/chosen": -2.1414482593536377,
      "logits/rejected": -2.112384796142578,
      "logps/chosen": -162.0722198486328,
      "logps/rejected": -151.81146240234375,
      "loss": 0.8236,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04249238967895508,
      "rewards/margins": -0.19530200958251953,
      "rewards/rejected": 0.2377944141626358,
      "step": 210
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.968431170340706e-05,
      "logits/chosen": -2.2804222106933594,
      "logits/rejected": -2.297311782836914,
      "logps/chosen": -130.9635772705078,
      "logps/rejected": -132.71075439453125,
      "loss": 0.7031,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.05266380310058594,
      "rewards/margins": 0.007896373979747295,
      "rewards/rejected": -0.06056017801165581,
      "step": 211
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.9678610277493275e-05,
      "logits/chosen": -2.324260711669922,
      "logits/rejected": -2.287720203399658,
      "logps/chosen": -144.84410095214844,
      "logps/rejected": -141.16293334960938,
      "loss": 0.693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13754111528396606,
      "rewards/margins": 0.018452219665050507,
      "rewards/rejected": 0.11908888816833496,
      "step": 212
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.967285815959759e-05,
      "logits/chosen": -2.1252338886260986,
      "logits/rejected": -2.181086778640747,
      "logps/chosen": -165.26145935058594,
      "logps/rejected": -185.48065185546875,
      "loss": 0.6365,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05688684061169624,
      "rewards/margins": 0.2325521856546402,
      "rewards/rejected": -0.17566533386707306,
      "step": 213
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.9667055361535354e-05,
      "logits/chosen": -2.0358633995056152,
      "logits/rejected": -2.037306308746338,
      "logps/chosen": -158.7928466796875,
      "logps/rejected": -174.2408905029297,
      "loss": 0.7384,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20100197196006775,
      "rewards/margins": -0.03293357789516449,
      "rewards/rejected": -0.16806840896606445,
      "step": 214
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.9661201895226e-05,
      "logits/chosen": -2.1538403034210205,
      "logits/rejected": -2.2139194011688232,
      "logps/chosen": -131.5411376953125,
      "logps/rejected": -151.45513916015625,
      "loss": 0.6839,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008547332137823105,
      "rewards/margins": 0.0680898129940033,
      "rewards/rejected": -0.059542469680309296,
      "step": 215
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.965529777269306e-05,
      "logits/chosen": -2.2412662506103516,
      "logits/rejected": -2.2856147289276123,
      "logps/chosen": -117.49397277832031,
      "logps/rejected": -120.62272644042969,
      "loss": 0.6965,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.228101447224617,
      "rewards/margins": 0.008968396112322807,
      "rewards/rejected": -0.23706983029842377,
      "step": 216
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.964934300606411e-05,
      "logits/chosen": -2.066718816757202,
      "logits/rejected": -2.138875722885132,
      "logps/chosen": -153.00021362304688,
      "logps/rejected": -164.0558624267578,
      "loss": 0.6836,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10881123691797256,
      "rewards/margins": 0.06324261426925659,
      "rewards/rejected": -0.17205384373664856,
      "step": 217
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.964333760757074e-05,
      "logits/chosen": -2.3143880367279053,
      "logits/rejected": -2.184577703475952,
      "logps/chosen": -161.26583862304688,
      "logps/rejected": -178.37884521484375,
      "loss": 0.6808,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.09615574032068253,
      "rewards/margins": 0.0893949344754219,
      "rewards/rejected": -0.18555067479610443,
      "step": 218
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.963728158954856e-05,
      "logits/chosen": -2.3663442134857178,
      "logits/rejected": -2.404465675354004,
      "logps/chosen": -140.4896240234375,
      "logps/rejected": -160.0462188720703,
      "loss": 0.6943,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.03150591999292374,
      "rewards/margins": 0.02258572168648243,
      "rewards/rejected": -0.05409163981676102,
      "step": 219
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.963117496443715e-05,
      "logits/chosen": -2.264538526535034,
      "logits/rejected": -2.2544782161712646,
      "logps/chosen": -180.01846313476562,
      "logps/rejected": -177.05178833007812,
      "loss": 0.7132,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13371305167675018,
      "rewards/margins": -0.008653441444039345,
      "rewards/rejected": -0.1250596046447754,
      "step": 220
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.9625017744780045e-05,
      "logits/chosen": -2.1741209030151367,
      "logits/rejected": -2.1468567848205566,
      "logps/chosen": -164.97640991210938,
      "logps/rejected": -169.90350341796875,
      "loss": 0.6725,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10341596603393555,
      "rewards/margins": 0.06766539812088013,
      "rewards/rejected": -0.17108136415481567,
      "step": 221
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.96188099432247e-05,
      "logits/chosen": -2.02036714553833,
      "logits/rejected": -2.0622000694274902,
      "logps/chosen": -196.87257385253906,
      "logps/rejected": -192.37307739257812,
      "loss": 0.692,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.21568839251995087,
      "rewards/margins": 0.0388670451939106,
      "rewards/rejected": -0.25455543398857117,
      "step": 222
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.9612551572522464e-05,
      "logits/chosen": -2.2688815593719482,
      "logits/rejected": -2.305769920349121,
      "logps/chosen": -143.64569091796875,
      "logps/rejected": -154.7953338623047,
      "loss": 0.6345,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16745011508464813,
      "rewards/margins": 0.1968405842781067,
      "rewards/rejected": -0.36429068446159363,
      "step": 223
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.960624264552858e-05,
      "logits/chosen": -2.271151065826416,
      "logits/rejected": -2.328986167907715,
      "logps/chosen": -133.5254364013672,
      "logps/rejected": -172.62962341308594,
      "loss": 0.7838,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2519568204879761,
      "rewards/margins": -0.10307664424180984,
      "rewards/rejected": -0.14888018369674683,
      "step": 224
    },
    {
      "epoch": 0.29,
      "learning_rate": 4.9599883175202124e-05,
      "logits/chosen": -2.3042290210723877,
      "logits/rejected": -2.285111427307129,
      "logps/chosen": -114.66839599609375,
      "logps/rejected": -120.16878509521484,
      "loss": 0.6911,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0025178715586662292,
      "rewards/margins": 0.055260900408029556,
      "rewards/rejected": -0.052743006497621536,
      "step": 225
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.9593473174605974e-05,
      "logits/chosen": -2.4031026363372803,
      "logits/rejected": -2.3971669673919678,
      "logps/chosen": -180.76190185546875,
      "logps/rejected": -189.8751220703125,
      "loss": 0.7679,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2216603308916092,
      "rewards/margins": -0.037253011018037796,
      "rewards/rejected": -0.1844073235988617,
      "step": 226
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.958701265690685e-05,
      "logits/chosen": -2.2369589805603027,
      "logits/rejected": -2.213113784790039,
      "logps/chosen": -163.1219482421875,
      "logps/rejected": -163.7172393798828,
      "loss": 0.7187,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.29494044184684753,
      "rewards/margins": -0.002360118553042412,
      "rewards/rejected": -0.29258033633232117,
      "step": 227
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.958050163537519e-05,
      "logits/chosen": -2.4455294609069824,
      "logits/rejected": -2.393801212310791,
      "logps/chosen": -143.6052703857422,
      "logps/rejected": -128.57278442382812,
      "loss": 0.7247,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2546530067920685,
      "rewards/margins": -0.035662561655044556,
      "rewards/rejected": -0.2189904749393463,
      "step": 228
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.957394012338519e-05,
      "logits/chosen": -2.242324113845825,
      "logits/rejected": -2.276301383972168,
      "logps/chosen": -154.13751220703125,
      "logps/rejected": -155.56005859375,
      "loss": 0.75,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3772943615913391,
      "rewards/margins": -0.08221397548913956,
      "rewards/rejected": -0.29508039355278015,
      "step": 229
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.956732813441477e-05,
      "logits/chosen": -2.4516844749450684,
      "logits/rejected": -2.397124767303467,
      "logps/chosen": -142.58702087402344,
      "logps/rejected": -139.663330078125,
      "loss": 0.8001,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2911987900733948,
      "rewards/margins": -0.15551666915416718,
      "rewards/rejected": -0.1356821358203888,
      "step": 230
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.956066568204552e-05,
      "logits/chosen": -2.039741039276123,
      "logits/rejected": -2.023181200027466,
      "logps/chosen": -154.21315002441406,
      "logps/rejected": -135.28794860839844,
      "loss": 0.6237,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0628127008676529,
      "rewards/margins": 0.22161367535591125,
      "rewards/rejected": -0.28442639112472534,
      "step": 231
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.955395277996268e-05,
      "logits/chosen": -2.2360501289367676,
      "logits/rejected": -2.2908437252044678,
      "logps/chosen": -184.1702423095703,
      "logps/rejected": -185.9180145263672,
      "loss": 0.6513,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20668905973434448,
      "rewards/margins": 0.1489761769771576,
      "rewards/rejected": -0.3556652069091797,
      "step": 232
    },
    {
      "epoch": 0.3,
      "learning_rate": 4.954718944195512e-05,
      "logits/chosen": -2.2392733097076416,
      "logits/rejected": -2.256279706954956,
      "logps/chosen": -140.51260375976562,
      "logps/rejected": -144.2230682373047,
      "loss": 0.6254,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18107274174690247,
      "rewards/margins": 0.16339734196662903,
      "rewards/rejected": -0.3444700539112091,
      "step": 233
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.954037568191534e-05,
      "logits/chosen": -2.2857413291931152,
      "logits/rejected": -2.2823734283447266,
      "logps/chosen": -139.91226196289062,
      "logps/rejected": -140.5911407470703,
      "loss": 0.6551,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10547606647014618,
      "rewards/margins": 0.11846562474966049,
      "rewards/rejected": -0.22394171357154846,
      "step": 234
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.9533511513839384e-05,
      "logits/chosen": -2.052267551422119,
      "logits/rejected": -2.02097487449646,
      "logps/chosen": -135.19448852539062,
      "logps/rejected": -150.51913452148438,
      "loss": 0.678,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1995100975036621,
      "rewards/margins": 0.07375679910182953,
      "rewards/rejected": -0.27326688170433044,
      "step": 235
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.9526596951826824e-05,
      "logits/chosen": -2.1060378551483154,
      "logits/rejected": -2.1363437175750732,
      "logps/chosen": -189.6680145263672,
      "logps/rejected": -194.1062774658203,
      "loss": 0.7673,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.20259276032447815,
      "rewards/margins": -0.09232301265001297,
      "rewards/rejected": -0.11026974767446518,
      "step": 236
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.951963201008076e-05,
      "logits/chosen": -2.146965265274048,
      "logits/rejected": -2.1230671405792236,
      "logps/chosen": -161.82183837890625,
      "logps/rejected": -152.2267303466797,
      "loss": 0.785,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.17847128212451935,
      "rewards/margins": -0.1518484354019165,
      "rewards/rejected": -0.026622820645570755,
      "step": 237
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.951261670290781e-05,
      "logits/chosen": -2.0150346755981445,
      "logits/rejected": -2.04280424118042,
      "logps/chosen": -204.8814239501953,
      "logps/rejected": -211.47410583496094,
      "loss": 0.8185,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.35267120599746704,
      "rewards/margins": -0.18207934498786926,
      "rewards/rejected": -0.17059186100959778,
      "step": 238
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.950555104471799e-05,
      "logits/chosen": -2.0696749687194824,
      "logits/rejected": -2.0690653324127197,
      "logps/chosen": -180.72787475585938,
      "logps/rejected": -167.14132690429688,
      "loss": 0.701,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1277678906917572,
      "rewards/margins": 0.01966879889369011,
      "rewards/rejected": -0.14743672311306,
      "step": 239
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.949843505002477e-05,
      "logits/chosen": -2.276556968688965,
      "logits/rejected": -2.307004928588867,
      "logps/chosen": -139.89166259765625,
      "logps/rejected": -149.29598999023438,
      "loss": 0.5977,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11064109206199646,
      "rewards/margins": 0.22020389139652252,
      "rewards/rejected": -0.10956278443336487,
      "step": 240
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.9491268733445034e-05,
      "logits/chosen": -2.3263444900512695,
      "logits/rejected": -2.216820240020752,
      "logps/chosen": -176.23760986328125,
      "logps/rejected": -181.00119018554688,
      "loss": 0.5842,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14714036881923676,
      "rewards/margins": 0.2764008939266205,
      "rewards/rejected": -0.4235413074493408,
      "step": 241
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.9484052109698984e-05,
      "logits/chosen": -2.2672414779663086,
      "logits/rejected": -2.29109525680542,
      "logps/chosen": -146.66981506347656,
      "logps/rejected": -148.75990295410156,
      "loss": 0.6706,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.010637687519192696,
      "rewards/margins": 0.07865148037672043,
      "rewards/rejected": -0.08928915858268738,
      "step": 242
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.947678519361021e-05,
      "logits/chosen": -2.173220634460449,
      "logits/rejected": -2.1617910861968994,
      "logps/chosen": -157.75787353515625,
      "logps/rejected": -161.2376251220703,
      "loss": 0.7178,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1290948987007141,
      "rewards/margins": 0.04089619964361191,
      "rewards/rejected": -0.16999109089374542,
      "step": 243
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.946946800010556e-05,
      "logits/chosen": -2.089167833328247,
      "logits/rejected": -2.098552942276001,
      "logps/chosen": -167.84510803222656,
      "logps/rejected": -177.17660522460938,
      "loss": 0.7749,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.23455895483493805,
      "rewards/margins": -0.11604375392198563,
      "rewards/rejected": -0.11851520091295242,
      "step": 244
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.946210054421518e-05,
      "logits/chosen": -2.3614742755889893,
      "logits/rejected": -2.367582321166992,
      "logps/chosen": -138.63021850585938,
      "logps/rejected": -138.73843383789062,
      "loss": 0.7186,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0001854933798313141,
      "rewards/margins": -0.002179570496082306,
      "rewards/rejected": 0.001994088292121887,
      "step": 245
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.945468284107246e-05,
      "logits/chosen": -2.033447265625,
      "logits/rejected": -2.047463893890381,
      "logps/chosen": -163.36306762695312,
      "logps/rejected": -178.01649475097656,
      "loss": 0.6706,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.026612814515829086,
      "rewards/margins": 0.12761405110359192,
      "rewards/rejected": -0.10100121051073074,
      "step": 246
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.944721490591401e-05,
      "logits/chosen": -2.2349605560302734,
      "logits/rejected": -2.2344400882720947,
      "logps/chosen": -153.67425537109375,
      "logps/rejected": -140.24844360351562,
      "loss": 0.7657,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.08484932780265808,
      "rewards/margins": -0.11917441338300705,
      "rewards/rejected": 0.034325070679187775,
      "step": 247
    },
    {
      "epoch": 0.32,
      "learning_rate": 4.9439696754079595e-05,
      "logits/chosen": -2.026840925216675,
      "logits/rejected": -2.0905795097351074,
      "logps/chosen": -153.32894897460938,
      "logps/rejected": -157.39907836914062,
      "loss": 0.609,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1143321692943573,
      "rewards/margins": 0.21959424018859863,
      "rewards/rejected": -0.10526210069656372,
      "step": 248
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9432128401012144e-05,
      "logits/chosen": -2.2150955200195312,
      "logits/rejected": -2.1640408039093018,
      "logps/chosen": -176.57774353027344,
      "logps/rejected": -177.27870178222656,
      "loss": 0.8383,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.25025683641433716,
      "rewards/margins": -0.19921274483203888,
      "rewards/rejected": -0.05104408413171768,
      "step": 249
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9424509862257706e-05,
      "logits/chosen": -1.9792041778564453,
      "logits/rejected": -1.9765446186065674,
      "logps/chosen": -134.66946411132812,
      "logps/rejected": -144.38739013671875,
      "loss": 0.6723,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.0672139897942543,
      "rewards/margins": 0.08019405603408813,
      "rewards/rejected": -0.14740802347660065,
      "step": 250
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.941684115346541e-05,
      "logits/chosen": -2.319556474685669,
      "logits/rejected": -2.351644277572632,
      "logps/chosen": -173.3380126953125,
      "logps/rejected": -170.59963989257812,
      "loss": 0.7212,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.019015267491340637,
      "rewards/margins": -0.03376225382089615,
      "rewards/rejected": 0.05277752876281738,
      "step": 251
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.940912229038745e-05,
      "logits/chosen": -2.0435500144958496,
      "logits/rejected": -2.022526502609253,
      "logps/chosen": -135.12265014648438,
      "logps/rejected": -125.80049133300781,
      "loss": 0.7075,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.021306686103343964,
      "rewards/margins": 0.02372751012444496,
      "rewards/rejected": -0.045034196227788925,
      "step": 252
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9401353288879024e-05,
      "logits/chosen": -1.973745584487915,
      "logits/rejected": -1.964308738708496,
      "logps/chosen": -148.7356719970703,
      "logps/rejected": -140.4853515625,
      "loss": 0.9136,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.35221704840660095,
      "rewards/margins": -0.27745121717453003,
      "rewards/rejected": -0.07476583123207092,
      "step": 253
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9393534164898335e-05,
      "logits/chosen": -2.2897889614105225,
      "logits/rejected": -2.2667198181152344,
      "logps/chosen": -169.73867797851562,
      "logps/rejected": -154.4757080078125,
      "loss": 0.7961,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.01284090057015419,
      "rewards/margins": -0.14840884506702423,
      "rewards/rejected": 0.13556794822216034,
      "step": 254
    },
    {
      "epoch": 0.33,
      "learning_rate": 4.9385664934506526e-05,
      "logits/chosen": -2.320432186126709,
      "logits/rejected": -2.2988882064819336,
      "logps/chosen": -144.0506591796875,
      "logps/rejected": -145.1551055908203,
      "loss": 0.8377,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.11610221862792969,
      "rewards/margins": -0.22257395088672638,
      "rewards/rejected": 0.33867618441581726,
      "step": 255
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.937774561386768e-05,
      "logits/chosen": -2.1970014572143555,
      "logits/rejected": -2.222613573074341,
      "logps/chosen": -138.17433166503906,
      "logps/rejected": -143.5152130126953,
      "loss": 0.7075,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0024851299822330475,
      "rewards/margins": 0.022421889007091522,
      "rewards/rejected": -0.01993674784898758,
      "step": 256
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.936977621924875e-05,
      "logits/chosen": -2.1020843982696533,
      "logits/rejected": -2.0904481410980225,
      "logps/chosen": -148.5655517578125,
      "logps/rejected": -153.73941040039062,
      "loss": 0.6322,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.003152729943394661,
      "rewards/margins": 0.18040181696414948,
      "rewards/rejected": -0.17724908888339996,
      "step": 257
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.9361756767019564e-05,
      "logits/chosen": -1.8264960050582886,
      "logits/rejected": -1.792884349822998,
      "logps/chosen": -168.45526123046875,
      "logps/rejected": -160.93218994140625,
      "loss": 0.7427,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.034992799162864685,
      "rewards/margins": -0.04541083052754402,
      "rewards/rejected": 0.010418036952614784,
      "step": 258
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.935368727365276e-05,
      "logits/chosen": -2.033273458480835,
      "logits/rejected": -2.0581772327423096,
      "logps/chosen": -151.52731323242188,
      "logps/rejected": -166.76388549804688,
      "loss": 0.7695,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.07278600335121155,
      "rewards/margins": -0.08771771937608719,
      "rewards/rejected": 0.014931721612811089,
      "step": 259
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.934556775572377e-05,
      "logits/chosen": -2.1122307777404785,
      "logits/rejected": -2.1686928272247314,
      "logps/chosen": -158.33444213867188,
      "logps/rejected": -155.8374481201172,
      "loss": 0.7004,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11077702045440674,
      "rewards/margins": 0.048048801720142365,
      "rewards/rejected": -0.1588258296251297,
      "step": 260
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.9337398229910784e-05,
      "logits/chosen": -2.0639896392822266,
      "logits/rejected": -2.0493862628936768,
      "logps/chosen": -140.65463256835938,
      "logps/rejected": -143.4046173095703,
      "loss": 0.6137,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.332168847322464,
      "rewards/margins": 0.21151331067085266,
      "rewards/rejected": 0.12065552175045013,
      "step": 261
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.932917871299471e-05,
      "logits/chosen": -2.1873834133148193,
      "logits/rejected": -2.1482584476470947,
      "logps/chosen": -156.8861846923828,
      "logps/rejected": -152.4180450439453,
      "loss": 0.7677,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.044190216809511185,
      "rewards/margins": -0.10401745140552521,
      "rewards/rejected": 0.1482076644897461,
      "step": 262
    },
    {
      "epoch": 0.34,
      "learning_rate": 4.9320909221859134e-05,
      "logits/chosen": -2.1388399600982666,
      "logits/rejected": -2.1468729972839355,
      "logps/chosen": -150.1500701904297,
      "logps/rejected": -152.03726196289062,
      "loss": 0.7504,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.08211887627840042,
      "rewards/margins": -0.07361260801553726,
      "rewards/rejected": 0.15573148429393768,
      "step": 263
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.9312589773490304e-05,
      "logits/chosen": -1.996654987335205,
      "logits/rejected": -1.9872137308120728,
      "logps/chosen": -150.1443328857422,
      "logps/rejected": -155.907958984375,
      "loss": 0.7484,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.17925025522708893,
      "rewards/margins": -0.05667828768491745,
      "rewards/rejected": 0.23592855036258698,
      "step": 264
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.930422038497708e-05,
      "logits/chosen": -2.2004756927490234,
      "logits/rejected": -2.2537848949432373,
      "logps/chosen": -150.181640625,
      "logps/rejected": -177.4576416015625,
      "loss": 0.6491,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.28161391615867615,
      "rewards/margins": 0.11887946724891663,
      "rewards/rejected": 0.16273444890975952,
      "step": 265
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.92958010735109e-05,
      "logits/chosen": -2.1067912578582764,
      "logits/rejected": -2.1667635440826416,
      "logps/chosen": -122.0634765625,
      "logps/rejected": -125.26715850830078,
      "loss": 0.7781,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.10816986113786697,
      "rewards/margins": -0.11183619499206543,
      "rewards/rejected": 0.2200060486793518,
      "step": 266
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.928733185638575e-05,
      "logits/chosen": -2.1284282207489014,
      "logits/rejected": -2.1388614177703857,
      "logps/chosen": -145.78765869140625,
      "logps/rejected": -164.8605194091797,
      "loss": 0.694,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06616201996803284,
      "rewards/margins": 0.05039716139435768,
      "rewards/rejected": 0.01576484926044941,
      "step": 267
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.927881275099815e-05,
      "logits/chosen": -2.2518081665039062,
      "logits/rejected": -2.2290408611297607,
      "logps/chosen": -153.83200073242188,
      "logps/rejected": -160.59776306152344,
      "loss": 0.7557,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.1310650259256363,
      "rewards/margins": -0.07667630910873413,
      "rewards/rejected": 0.20774134993553162,
      "step": 268
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.927024377484705e-05,
      "logits/chosen": -2.0690414905548096,
      "logits/rejected": -2.096713066101074,
      "logps/chosen": -168.7005157470703,
      "logps/rejected": -168.45872497558594,
      "loss": 0.6711,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.16334614157676697,
      "rewards/margins": 0.09711476415395737,
      "rewards/rejected": 0.06623139977455139,
      "step": 269
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.9261624945533855e-05,
      "logits/chosen": -1.6682740449905396,
      "logits/rejected": -1.611401915550232,
      "logps/chosen": -170.30328369140625,
      "logps/rejected": -174.10769653320312,
      "loss": 0.8147,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.10478762537240982,
      "rewards/margins": -0.1818379908800125,
      "rewards/rejected": 0.07705036550760269,
      "step": 270
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.925295628076241e-05,
      "logits/chosen": -1.9349985122680664,
      "logits/rejected": -1.9530787467956543,
      "logps/chosen": -130.31883239746094,
      "logps/rejected": -152.9124755859375,
      "loss": 0.8018,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0537966787815094,
      "rewards/margins": -0.1305672526359558,
      "rewards/rejected": 0.1843639612197876,
      "step": 271
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.9244237798338866e-05,
      "logits/chosen": -2.0011062622070312,
      "logits/rejected": -1.9965519905090332,
      "logps/chosen": -164.77838134765625,
      "logps/rejected": -192.95458984375,
      "loss": 0.7462,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0018954463303089142,
      "rewards/margins": -0.058521248400211334,
      "rewards/rejected": 0.056625787168741226,
      "step": 272
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.923546951617175e-05,
      "logits/chosen": -2.117265224456787,
      "logits/rejected": -2.1309285163879395,
      "logps/chosen": -157.73707580566406,
      "logps/rejected": -156.6586456298828,
      "loss": 0.7046,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09399585425853729,
      "rewards/margins": 0.057814061641693115,
      "rewards/rejected": 0.03618178144097328,
      "step": 273
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.922665145227187e-05,
      "logits/chosen": -2.2057220935821533,
      "logits/rejected": -2.1393046379089355,
      "logps/chosen": -182.5475311279297,
      "logps/rejected": -164.41293334960938,
      "loss": 0.8501,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0016194283962249756,
      "rewards/margins": -0.25813543796539307,
      "rewards/rejected": 0.2565160095691681,
      "step": 274
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.9217783624752266e-05,
      "logits/chosen": -2.2684426307678223,
      "logits/rejected": -2.243990659713745,
      "logps/chosen": -128.5909881591797,
      "logps/rejected": -125.42144775390625,
      "loss": 0.6339,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.07768978923559189,
      "rewards/margins": 0.20721395313739777,
      "rewards/rejected": -0.12952415645122528,
      "step": 275
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.920886605182823e-05,
      "logits/chosen": -2.2352826595306396,
      "logits/rejected": -2.317643404006958,
      "logps/chosen": -147.9606170654297,
      "logps/rejected": -151.64022827148438,
      "loss": 0.7825,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.1962936520576477,
      "rewards/margins": -0.0966653823852539,
      "rewards/rejected": 0.2929590344429016,
      "step": 276
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.919989875181722e-05,
      "logits/chosen": -2.0506582260131836,
      "logits/rejected": -2.056596517562866,
      "logps/chosen": -133.77655029296875,
      "logps/rejected": -142.32601928710938,
      "loss": 0.698,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06665589660406113,
      "rewards/margins": 0.017916321754455566,
      "rewards/rejected": 0.04873957484960556,
      "step": 277
    },
    {
      "epoch": 0.36,
      "learning_rate": 4.919088174313884e-05,
      "logits/chosen": -2.1315078735351562,
      "logits/rejected": -2.1690430641174316,
      "logps/chosen": -166.51275634765625,
      "logps/rejected": -171.87623596191406,
      "loss": 0.6526,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.17828050255775452,
      "rewards/margins": 0.1265142858028412,
      "rewards/rejected": 0.05176621302962303,
      "step": 278
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.91818150443148e-05,
      "logits/chosen": -2.316528797149658,
      "logits/rejected": -2.3489573001861572,
      "logps/chosen": -158.4711456298828,
      "logps/rejected": -156.23777770996094,
      "loss": 0.6217,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.19824568927288055,
      "rewards/margins": 0.19765932857990265,
      "rewards/rejected": 0.0005863434635102749,
      "step": 279
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.917269867396886e-05,
      "logits/chosen": -2.1244661808013916,
      "logits/rejected": -2.0779430866241455,
      "logps/chosen": -164.66851806640625,
      "logps/rejected": -145.93212890625,
      "loss": 0.6832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0432266928255558,
      "rewards/margins": 0.07876121997833252,
      "rewards/rejected": -0.03553451970219612,
      "step": 280
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.916353265082686e-05,
      "logits/chosen": -2.397843837738037,
      "logits/rejected": -2.4166927337646484,
      "logps/chosen": -214.45030212402344,
      "logps/rejected": -215.8428497314453,
      "loss": 0.7811,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.009777067229151726,
      "rewards/margins": -0.09677901118993759,
      "rewards/rejected": 0.10655608028173447,
      "step": 281
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9154316993716565e-05,
      "logits/chosen": -2.245692491531372,
      "logits/rejected": -2.245835781097412,
      "logps/chosen": -156.67173767089844,
      "logps/rejected": -149.71917724609375,
      "loss": 0.7229,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05182276666164398,
      "rewards/margins": -0.009018277749419212,
      "rewards/rejected": 0.06084103882312775,
      "step": 282
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9145051721567734e-05,
      "logits/chosen": -2.2843127250671387,
      "logits/rejected": -2.2395739555358887,
      "logps/chosen": -167.3928985595703,
      "logps/rejected": -162.44593811035156,
      "loss": 0.7401,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.057207878679037094,
      "rewards/margins": -0.031176520511507988,
      "rewards/rejected": 0.08838438242673874,
      "step": 283
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.913573685341205e-05,
      "logits/chosen": -2.0592617988586426,
      "logits/rejected": -2.108931064605713,
      "logps/chosen": -189.07752990722656,
      "logps/rejected": -181.4488983154297,
      "loss": 0.8664,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3183850646018982,
      "rewards/margins": -0.25120943784713745,
      "rewards/rejected": -0.06717558205127716,
      "step": 284
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.9126372408383025e-05,
      "logits/chosen": -1.9963423013687134,
      "logits/rejected": -1.9505279064178467,
      "logps/chosen": -149.78964233398438,
      "logps/rejected": -148.13140869140625,
      "loss": 0.7801,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.006365638226270676,
      "rewards/margins": -0.0903141051530838,
      "rewards/rejected": 0.09667973965406418,
      "step": 285
    },
    {
      "epoch": 0.37,
      "learning_rate": 4.911695840571605e-05,
      "logits/chosen": -2.259364604949951,
      "logits/rejected": -2.2460901737213135,
      "logps/chosen": -173.76356506347656,
      "logps/rejected": -162.274169921875,
      "loss": 0.7328,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.26215800642967224,
      "rewards/margins": -0.027749449014663696,
      "rewards/rejected": -0.23440855741500854,
      "step": 286
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.910749486474828e-05,
      "logits/chosen": -2.1809744834899902,
      "logits/rejected": -2.1365909576416016,
      "logps/chosen": -173.26553344726562,
      "logps/rejected": -168.60324096679688,
      "loss": 0.6541,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13890515267848969,
      "rewards/margins": 0.1398504674434662,
      "rewards/rejected": -0.2787555754184723,
      "step": 287
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.909798180491865e-05,
      "logits/chosen": -2.2078680992126465,
      "logits/rejected": -2.202317714691162,
      "logps/chosen": -161.5787353515625,
      "logps/rejected": -148.63571166992188,
      "loss": 0.6944,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0784807801246643,
      "rewards/margins": 0.04119132459163666,
      "rewards/rejected": 0.037289444357156754,
      "step": 288
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.9088419245767803e-05,
      "logits/chosen": -2.323357582092285,
      "logits/rejected": -2.331782817840576,
      "logps/chosen": -154.3722686767578,
      "logps/rejected": -142.60247802734375,
      "loss": 0.7108,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.03877534344792366,
      "rewards/margins": 0.031510498374700546,
      "rewards/rejected": 0.007264849729835987,
      "step": 289
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.907880720693804e-05,
      "logits/chosen": -2.3022680282592773,
      "logits/rejected": -2.242792844772339,
      "logps/chosen": -143.08497619628906,
      "logps/rejected": -142.6212158203125,
      "loss": 0.6595,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.19559280574321747,
      "rewards/margins": 0.14293703436851501,
      "rewards/rejected": 0.052655745297670364,
      "step": 290
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.9069145708173324e-05,
      "logits/chosen": -2.24276065826416,
      "logits/rejected": -2.2772743701934814,
      "logps/chosen": -141.13607788085938,
      "logps/rejected": -143.68528747558594,
      "loss": 0.749,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13126200437545776,
      "rewards/margins": -0.08278003334999084,
      "rewards/rejected": -0.04848198592662811,
      "step": 291
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.9059434769319205e-05,
      "logits/chosen": -1.9449436664581299,
      "logits/rejected": -1.8548604249954224,
      "logps/chosen": -182.99716186523438,
      "logps/rejected": -157.7788848876953,
      "loss": 0.7759,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.11397290229797363,
      "rewards/margins": -0.0827551856637001,
      "rewards/rejected": -0.031217724084854126,
      "step": 292
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.904967441032278e-05,
      "logits/chosen": -2.3559162616729736,
      "logits/rejected": -2.351862907409668,
      "logps/chosen": -156.9031982421875,
      "logps/rejected": -151.17758178710938,
      "loss": 0.7666,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.030413679778575897,
      "rewards/margins": -0.07400794327259064,
      "rewards/rejected": 0.04359426349401474,
      "step": 293
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.903986465123266e-05,
      "logits/chosen": -2.013231039047241,
      "logits/rejected": -2.1264541149139404,
      "logps/chosen": -142.36181640625,
      "logps/rejected": -151.9750213623047,
      "loss": 0.7971,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.08235807716846466,
      "rewards/margins": -0.12663468718528748,
      "rewards/rejected": 0.04427662491798401,
      "step": 294
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.903000551219894e-05,
      "logits/chosen": -2.1607303619384766,
      "logits/rejected": -2.1577768325805664,
      "logps/chosen": -180.66165161132812,
      "logps/rejected": -193.0799102783203,
      "loss": 0.6103,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06296667456626892,
      "rewards/margins": 0.2222498655319214,
      "rewards/rejected": -0.15928319096565247,
      "step": 295
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.902009701347313e-05,
      "logits/chosen": -2.039292335510254,
      "logits/rejected": -2.075709342956543,
      "logps/chosen": -139.63912963867188,
      "logps/rejected": -145.09732055664062,
      "loss": 0.7106,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06535654515028,
      "rewards/margins": 0.01611308380961418,
      "rewards/rejected": -0.08146963268518448,
      "step": 296
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.901013917540814e-05,
      "logits/chosen": -2.17149019241333,
      "logits/rejected": -2.1316046714782715,
      "logps/chosen": -149.57943725585938,
      "logps/rejected": -135.62716674804688,
      "loss": 0.6599,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.09369993209838867,
      "rewards/margins": 0.16279715299606323,
      "rewards/rejected": -0.06909724324941635,
      "step": 297
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.900013201845821e-05,
      "logits/chosen": -2.245067834854126,
      "logits/rejected": -2.3142921924591064,
      "logps/chosen": -249.42477416992188,
      "logps/rejected": -244.9073486328125,
      "loss": 0.6299,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03764531761407852,
      "rewards/margins": 0.16476468741893768,
      "rewards/rejected": -0.2024100124835968,
      "step": 298
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.899007556317893e-05,
      "logits/chosen": -2.1344449520111084,
      "logits/rejected": -2.114758014678955,
      "logps/chosen": -149.18663024902344,
      "logps/rejected": -157.50579833984375,
      "loss": 0.7001,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.05027259886264801,
      "rewards/margins": 0.06326065212488174,
      "rewards/rejected": -0.11353327333927155,
      "step": 299
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.8979969830227086e-05,
      "logits/chosen": -2.040646553039551,
      "logits/rejected": -2.045273780822754,
      "logps/chosen": -202.0292510986328,
      "logps/rejected": -214.07327270507812,
      "loss": 0.6409,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.050717130303382874,
      "rewards/margins": 0.17365548014640808,
      "rewards/rejected": -0.22437259554862976,
      "step": 300
    },
    {
      "epoch": 0.39,
      "learning_rate": 4.896981484036074e-05,
      "logits/chosen": -2.013162612915039,
      "logits/rejected": -1.9795409440994263,
      "logps/chosen": -146.5740509033203,
      "logps/rejected": -149.60594177246094,
      "loss": 0.6182,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.060083091259002686,
      "rewards/margins": 0.2053542286157608,
      "rewards/rejected": -0.2654373347759247,
      "step": 301
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.895961061443911e-05,
      "logits/chosen": -2.287144184112549,
      "logits/rejected": -2.292527914047241,
      "logps/chosen": -157.79879760742188,
      "logps/rejected": -156.17396545410156,
      "loss": 0.693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018628746271133423,
      "rewards/margins": 0.05455555394291878,
      "rewards/rejected": -0.035926803946495056,
      "step": 302
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.894935717342255e-05,
      "logits/chosen": -1.8966100215911865,
      "logits/rejected": -1.9237083196640015,
      "logps/chosen": -139.45045471191406,
      "logps/rejected": -163.8885955810547,
      "loss": 0.7204,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2020660787820816,
      "rewards/margins": 0.042556531727313995,
      "rewards/rejected": -0.2446226328611374,
      "step": 303
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.8939054538372496e-05,
      "logits/chosen": -2.121617317199707,
      "logits/rejected": -2.1448612213134766,
      "logps/chosen": -237.39845275878906,
      "logps/rejected": -254.28359985351562,
      "loss": 0.7159,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.0699370950460434,
      "rewards/margins": 0.008224628865718842,
      "rewards/rejected": 0.06171245500445366,
      "step": 304
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.8928702730451456e-05,
      "logits/chosen": -1.949974775314331,
      "logits/rejected": -1.870169997215271,
      "logps/chosen": -153.4409942626953,
      "logps/rejected": -156.4421844482422,
      "loss": 0.7495,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.12804728746414185,
      "rewards/margins": -0.07111337780952454,
      "rewards/rejected": -0.0569339245557785,
      "step": 305
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.891830177092294e-05,
      "logits/chosen": -2.206598997116089,
      "logits/rejected": -2.1638033390045166,
      "logps/chosen": -151.28504943847656,
      "logps/rejected": -148.6924285888672,
      "loss": 0.6044,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.030383877456188202,
      "rewards/margins": 0.2497837245464325,
      "rewards/rejected": -0.2801675796508789,
      "step": 306
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.8907851681151396e-05,
      "logits/chosen": -1.9466733932495117,
      "logits/rejected": -2.0422286987304688,
      "logps/chosen": -147.58941650390625,
      "logps/rejected": -163.70912170410156,
      "loss": 0.7529,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.18631082773208618,
      "rewards/margins": -0.07437282055616379,
      "rewards/rejected": -0.1119379922747612,
      "step": 307
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.889735248260221e-05,
      "logits/chosen": -1.9457015991210938,
      "logits/rejected": -1.9883947372436523,
      "logps/chosen": -168.67608642578125,
      "logps/rejected": -175.51678466796875,
      "loss": 0.6777,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1923048198223114,
      "rewards/margins": 0.08353283256292343,
      "rewards/rejected": -0.27583765983581543,
      "step": 308
    },
    {
      "epoch": 0.4,
      "learning_rate": 4.8886804196841626e-05,
      "logits/chosen": -2.105576515197754,
      "logits/rejected": -2.0363874435424805,
      "logps/chosen": -154.8956756591797,
      "logps/rejected": -155.22312927246094,
      "loss": 0.7898,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.08425075560808182,
      "rewards/margins": -0.13373690843582153,
      "rewards/rejected": 0.04948614165186882,
      "step": 309
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.887620684553674e-05,
      "logits/chosen": -2.1795973777770996,
      "logits/rejected": -2.2052454948425293,
      "logps/chosen": -132.4536895751953,
      "logps/rejected": -134.7356719970703,
      "loss": 0.8224,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.002498343586921692,
      "rewards/margins": -0.15511931478977203,
      "rewards/rejected": 0.15761765837669373,
      "step": 310
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.886556045045542e-05,
      "logits/chosen": -2.068981409072876,
      "logits/rejected": -2.0851194858551025,
      "logps/chosen": -146.88949584960938,
      "logps/rejected": -162.4698028564453,
      "loss": 0.7558,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15774422883987427,
      "rewards/margins": -0.06368841230869293,
      "rewards/rejected": -0.09405580163002014,
      "step": 311
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.8854865033466275e-05,
      "logits/chosen": -1.7522428035736084,
      "logits/rejected": -1.729603886604309,
      "logps/chosen": -217.58309936523438,
      "logps/rejected": -244.42282104492188,
      "loss": 0.7803,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.23889069259166718,
      "rewards/margins": -0.09420409798622131,
      "rewards/rejected": -0.14468660950660706,
      "step": 312
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.88441206165386e-05,
      "logits/chosen": -2.1225035190582275,
      "logits/rejected": -2.0949532985687256,
      "logps/chosen": -177.24856567382812,
      "logps/rejected": -175.7102813720703,
      "loss": 0.7095,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.12725457549095154,
      "rewards/margins": 0.04284512996673584,
      "rewards/rejected": -0.17009973526000977,
      "step": 313
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.8833327221742356e-05,
      "logits/chosen": -2.0727334022521973,
      "logits/rejected": -2.134239435195923,
      "logps/chosen": -131.3041534423828,
      "logps/rejected": -136.77574157714844,
      "loss": 0.6715,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.037539076060056686,
      "rewards/margins": 0.08883160352706909,
      "rewards/rejected": -0.0512925386428833,
      "step": 314
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.88224848712481e-05,
      "logits/chosen": -1.953855276107788,
      "logits/rejected": -1.9414265155792236,
      "logps/chosen": -187.20455932617188,
      "logps/rejected": -181.2246551513672,
      "loss": 0.5929,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.1832740604877472,
      "rewards/margins": 0.2624707818031311,
      "rewards/rejected": -0.07919671386480331,
      "step": 315
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.881159358732694e-05,
      "logits/chosen": -1.8854598999023438,
      "logits/rejected": -1.8628448247909546,
      "logps/chosen": -156.7054443359375,
      "logps/rejected": -158.3913116455078,
      "loss": 0.6956,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.11108584702014923,
      "rewards/margins": 0.05867990851402283,
      "rewards/rejected": -0.16976574063301086,
      "step": 316
    },
    {
      "epoch": 0.41,
      "learning_rate": 4.8800653392350526e-05,
      "logits/chosen": -1.9753170013427734,
      "logits/rejected": -2.036269426345825,
      "logps/chosen": -162.7784423828125,
      "logps/rejected": -176.72535705566406,
      "loss": 0.6218,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.21797724068164825,
      "rewards/margins": 0.17236095666885376,
      "rewards/rejected": -0.3903381824493408,
      "step": 317
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.8789664308790936e-05,
      "logits/chosen": -1.8070260286331177,
      "logits/rejected": -1.8477225303649902,
      "logps/chosen": -221.89990234375,
      "logps/rejected": -199.6925048828125,
      "loss": 0.7534,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5042496919631958,
      "rewards/margins": 0.0008268915116786957,
      "rewards/rejected": -0.5050765872001648,
      "step": 318
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.8778626359220715e-05,
      "logits/chosen": -1.9517831802368164,
      "logits/rejected": -1.9943265914916992,
      "logps/chosen": -157.50552368164062,
      "logps/rejected": -189.5386962890625,
      "loss": 0.7944,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.07389702647924423,
      "rewards/margins": -0.1338617205619812,
      "rewards/rejected": 0.059964705258607864,
      "step": 319
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.8767539566312734e-05,
      "logits/chosen": -2.0427169799804688,
      "logits/rejected": -1.9722158908843994,
      "logps/chosen": -153.33877563476562,
      "logps/rejected": -152.97161865234375,
      "loss": 0.9077,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.18528270721435547,
      "rewards/margins": -0.3350525200366974,
      "rewards/rejected": 0.14976979792118073,
      "step": 320
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.875640395284023e-05,
      "logits/chosen": -2.2405035495758057,
      "logits/rejected": -2.180358648300171,
      "logps/chosen": -149.2640838623047,
      "logps/rejected": -144.21060180664062,
      "loss": 0.6886,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.029363110661506653,
      "rewards/margins": 0.07899504899978638,
      "rewards/rejected": -0.10835815221071243,
      "step": 321
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.874521954167671e-05,
      "logits/chosen": -2.171722888946533,
      "logits/rejected": -2.134547710418701,
      "logps/chosen": -142.7288818359375,
      "logps/rejected": -133.72573852539062,
      "loss": 0.7838,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.31318625807762146,
      "rewards/margins": -0.14739766716957092,
      "rewards/rejected": -0.16578857600688934,
      "step": 322
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.8733986355795905e-05,
      "logits/chosen": -1.8786143064498901,
      "logits/rejected": -1.8554753065109253,
      "logps/chosen": -202.17709350585938,
      "logps/rejected": -205.63951110839844,
      "loss": 0.6038,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.08190246671438217,
      "rewards/margins": 0.2607104182243347,
      "rewards/rejected": -0.17880797386169434,
      "step": 323
    },
    {
      "epoch": 0.42,
      "learning_rate": 4.8722704418271745e-05,
      "logits/chosen": -2.1331138610839844,
      "logits/rejected": -2.1795654296875,
      "logps/chosen": -140.4710693359375,
      "logps/rejected": -143.41455078125,
      "loss": 0.6648,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.11869597434997559,
      "rewards/margins": 0.0922723338007927,
      "rewards/rejected": -0.21096831560134888,
      "step": 324
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.871137375227829e-05,
      "logits/chosen": -1.9935777187347412,
      "logits/rejected": -1.9692142009735107,
      "logps/chosen": -297.49334716796875,
      "logps/rejected": -299.84173583984375,
      "loss": 0.7695,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12770779430866241,
      "rewards/margins": -0.043505311012268066,
      "rewards/rejected": -0.08420247584581375,
      "step": 325
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.869999438108971e-05,
      "logits/chosen": -2.244154691696167,
      "logits/rejected": -2.2609543800354004,
      "logps/chosen": -148.76113891601562,
      "logps/rejected": -137.73040771484375,
      "loss": 0.7324,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.16916662454605103,
      "rewards/margins": -0.04308079555630684,
      "rewards/rejected": -0.12608584761619568,
      "step": 326
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.8688566328080215e-05,
      "logits/chosen": -2.1407310962677,
      "logits/rejected": -2.119994640350342,
      "logps/chosen": -167.45089721679688,
      "logps/rejected": -159.93060302734375,
      "loss": 0.7649,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3510248363018036,
      "rewards/margins": -0.09942013025283813,
      "rewards/rejected": -0.25160470604896545,
      "step": 327
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.867708961672399e-05,
      "logits/chosen": -2.185452699661255,
      "logits/rejected": -2.1928515434265137,
      "logps/chosen": -186.91505432128906,
      "logps/rejected": -185.78733825683594,
      "loss": 0.7766,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.09309504181146622,
      "rewards/margins": -0.11240511387586594,
      "rewards/rejected": 0.01931007206439972,
      "step": 328
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.866556427059519e-05,
      "logits/chosen": -2.1376357078552246,
      "logits/rejected": -2.2185373306274414,
      "logps/chosen": -166.6249542236328,
      "logps/rejected": -159.41290283203125,
      "loss": 0.8817,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1689298152923584,
      "rewards/margins": -0.18555757403373718,
      "rewards/rejected": 0.016627788543701172,
      "step": 329
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.865399031336787e-05,
      "logits/chosen": -1.9547293186187744,
      "logits/rejected": -2.0602641105651855,
      "logps/chosen": -159.6194610595703,
      "logps/rejected": -187.00814819335938,
      "loss": 0.5742,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08318576216697693,
      "rewards/margins": 0.3107747733592987,
      "rewards/rejected": -0.22758902609348297,
      "step": 330
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.8642367768815936e-05,
      "logits/chosen": -2.290693998336792,
      "logits/rejected": -2.2612967491149902,
      "logps/chosen": -204.46893310546875,
      "logps/rejected": -196.72003173828125,
      "loss": 0.6234,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.20541870594024658,
      "rewards/margins": 0.22441859543323517,
      "rewards/rejected": -0.42983728647232056,
      "step": 331
    },
    {
      "epoch": 0.43,
      "learning_rate": 4.863069666081307e-05,
      "logits/chosen": -2.1638312339782715,
      "logits/rejected": -2.109403133392334,
      "logps/chosen": -138.51168823242188,
      "logps/rejected": -137.55712890625,
      "loss": 0.7573,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.020162537693977356,
      "rewards/margins": -0.048720985651016235,
      "rewards/rejected": 0.028558451682329178,
      "step": 332
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.861897701333274e-05,
      "logits/chosen": -2.2542800903320312,
      "logits/rejected": -2.258378744125366,
      "logps/chosen": -134.4961700439453,
      "logps/rejected": -148.88816833496094,
      "loss": 0.7321,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.294142484664917,
      "rewards/margins": -0.008809719234704971,
      "rewards/rejected": -0.2853327691555023,
      "step": 333
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.86072088504481e-05,
      "logits/chosen": -2.152188539505005,
      "logits/rejected": -2.193638324737549,
      "logps/chosen": -138.4483184814453,
      "logps/rejected": -143.88937377929688,
      "loss": 0.7268,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.020661018788814545,
      "rewards/margins": 0.026195645332336426,
      "rewards/rejected": -0.00553460419178009,
      "step": 334
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.859539219633199e-05,
      "logits/chosen": -2.293026924133301,
      "logits/rejected": -2.278402805328369,
      "logps/chosen": -163.4123077392578,
      "logps/rejected": -180.30471801757812,
      "loss": 0.7216,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.1497456282377243,
      "rewards/margins": -0.013850819319486618,
      "rewards/rejected": -0.1358948051929474,
      "step": 335
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.8583527075256804e-05,
      "logits/chosen": -2.0941126346588135,
      "logits/rejected": -2.131237745285034,
      "logps/chosen": -153.41043090820312,
      "logps/rejected": -145.23020935058594,
      "loss": 0.6753,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.00170879065990448,
      "rewards/margins": 0.09639380127191544,
      "rewards/rejected": -0.09810256958007812,
      "step": 336
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.857161351159454e-05,
      "logits/chosen": -1.8922992944717407,
      "logits/rejected": -1.7540662288665771,
      "logps/chosen": -142.38389587402344,
      "logps/rejected": -148.71514892578125,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02964305877685547,
      "rewards/margins": 0.07973094284534454,
      "rewards/rejected": -0.10937398672103882,
      "step": 337
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.8559651529816664e-05,
      "logits/chosen": -2.170006513595581,
      "logits/rejected": -2.202960252761841,
      "logps/chosen": -144.66648864746094,
      "logps/rejected": -136.111083984375,
      "loss": 0.7541,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14397282898426056,
      "rewards/margins": -0.059857327491045,
      "rewards/rejected": -0.08411550521850586,
      "step": 338
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.854764115449411e-05,
      "logits/chosen": -2.098104238510132,
      "logits/rejected": -2.05627179145813,
      "logps/chosen": -138.97332763671875,
      "logps/rejected": -143.80482482910156,
      "loss": 0.7632,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13671807944774628,
      "rewards/margins": -0.11267251521348953,
      "rewards/rejected": -0.024045560508966446,
      "step": 339
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.853558241029723e-05,
      "logits/chosen": -2.045684337615967,
      "logits/rejected": -2.010531425476074,
      "logps/chosen": -158.4129180908203,
      "logps/rejected": -169.3193817138672,
      "loss": 0.7891,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.13485963642597198,
      "rewards/margins": -0.10890495777130127,
      "rewards/rejected": -0.025954678654670715,
      "step": 340
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.8523475321995715e-05,
      "logits/chosen": -2.199786901473999,
      "logits/rejected": -2.0181658267974854,
      "logps/chosen": -189.12388610839844,
      "logps/rejected": -139.44427490234375,
      "loss": 0.7936,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2502678632736206,
      "rewards/margins": -0.15687265992164612,
      "rewards/rejected": -0.09339523315429688,
      "step": 341
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.8511319914458555e-05,
      "logits/chosen": -2.1136443614959717,
      "logits/rejected": -2.1628942489624023,
      "logps/chosen": -153.02317810058594,
      "logps/rejected": -147.45071411132812,
      "loss": 0.7746,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.15984635055065155,
      "rewards/margins": -0.07732124626636505,
      "rewards/rejected": -0.0825251117348671,
      "step": 342
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.849911621265401e-05,
      "logits/chosen": -2.0306332111358643,
      "logits/rejected": -2.0360593795776367,
      "logps/chosen": -153.94784545898438,
      "logps/rejected": -162.53622436523438,
      "loss": 0.7162,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1451529562473297,
      "rewards/margins": 0.013205336406826973,
      "rewards/rejected": -0.15835829079151154,
      "step": 343
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.848686424164953e-05,
      "logits/chosen": -2.1170244216918945,
      "logits/rejected": -2.0833611488342285,
      "logps/chosen": -140.57786560058594,
      "logps/rejected": -133.22264099121094,
      "loss": 0.6097,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15281759202480316,
      "rewards/margins": 0.23962977528572083,
      "rewards/rejected": -0.08681218326091766,
      "step": 344
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.84745640266117e-05,
      "logits/chosen": -2.1526424884796143,
      "logits/rejected": -2.1702723503112793,
      "logps/chosen": -152.15927124023438,
      "logps/rejected": -158.22642517089844,
      "loss": 0.6938,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.09753294289112091,
      "rewards/margins": 0.03131475672125816,
      "rewards/rejected": -0.12884768843650818,
      "step": 345
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.846221559280624e-05,
      "logits/chosen": -2.025162696838379,
      "logits/rejected": -2.069690704345703,
      "logps/chosen": -147.05039978027344,
      "logps/rejected": -162.94961547851562,
      "loss": 0.6505,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.15327642858028412,
      "rewards/margins": 0.11415009945631027,
      "rewards/rejected": 0.03912632539868355,
      "step": 346
    },
    {
      "epoch": 0.45,
      "learning_rate": 4.844981896559787e-05,
      "logits/chosen": -2.208263635635376,
      "logits/rejected": -2.176539659500122,
      "logps/chosen": -212.233154296875,
      "logps/rejected": -209.1885223388672,
      "loss": 0.6771,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.31684011220932007,
      "rewards/margins": 0.08533424139022827,
      "rewards/rejected": -0.40217435359954834,
      "step": 347
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.8437374170450344e-05,
      "logits/chosen": -2.2429494857788086,
      "logits/rejected": -2.2173945903778076,
      "logps/chosen": -199.77957153320312,
      "logps/rejected": -156.09060668945312,
      "loss": 0.6945,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.056611090898513794,
      "rewards/margins": 0.11540088802576065,
      "rewards/rejected": -0.17201198637485504,
      "step": 348
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.842488123292632e-05,
      "logits/chosen": -2.088970422744751,
      "logits/rejected": -2.0711379051208496,
      "logps/chosen": -161.3831787109375,
      "logps/rejected": -153.05728149414062,
      "loss": 0.6521,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.25492504239082336,
      "rewards/margins": 0.13332128524780273,
      "rewards/rejected": 0.12160372734069824,
      "step": 349
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.8412340178687374e-05,
      "logits/chosen": -2.0223379135131836,
      "logits/rejected": -2.0152769088745117,
      "logps/chosen": -131.5534210205078,
      "logps/rejected": -137.33941650390625,
      "loss": 0.6658,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.1717926263809204,
      "rewards/margins": 0.12639762461185455,
      "rewards/rejected": -0.29819023609161377,
      "step": 350
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.839975103349391e-05,
      "logits/chosen": -2.299764633178711,
      "logits/rejected": -2.2954838275909424,
      "logps/chosen": -177.5594024658203,
      "logps/rejected": -162.41683959960938,
      "loss": 0.6371,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12795095145702362,
      "rewards/margins": 0.15119165182113647,
      "rewards/rejected": -0.2791425883769989,
      "step": 351
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.8387113823205096e-05,
      "logits/chosen": -2.0409903526306152,
      "logits/rejected": -2.0102832317352295,
      "logps/chosen": -171.60342407226562,
      "logps/rejected": -162.114013671875,
      "loss": 0.7369,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04189925640821457,
      "rewards/margins": 0.03296327590942383,
      "rewards/rejected": -0.074862539768219,
      "step": 352
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.8374428573778864e-05,
      "logits/chosen": -2.1743125915527344,
      "logits/rejected": -2.2494208812713623,
      "logps/chosen": -154.51788330078125,
      "logps/rejected": -161.42127990722656,
      "loss": 0.7557,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.04235090687870979,
      "rewards/margins": -0.0858534649014473,
      "rewards/rejected": 0.1282043755054474,
      "step": 353
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.8361695311271795e-05,
      "logits/chosen": -1.9694074392318726,
      "logits/rejected": -1.9777883291244507,
      "logps/chosen": -172.72686767578125,
      "logps/rejected": -187.39076232910156,
      "loss": 0.6227,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23133322596549988,
      "rewards/margins": 0.19938969612121582,
      "rewards/rejected": -0.4307229220867157,
      "step": 354
    },
    {
      "epoch": 0.46,
      "learning_rate": 4.83489140618391e-05,
      "logits/chosen": -2.125094175338745,
      "logits/rejected": -2.069443464279175,
      "logps/chosen": -177.20260620117188,
      "logps/rejected": -166.99986267089844,
      "loss": 0.681,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.07136122137308121,
      "rewards/margins": 0.09348535537719727,
      "rewards/rejected": -0.02212415263056755,
      "step": 355
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.833608485173457e-05,
      "logits/chosen": -2.26041841506958,
      "logits/rejected": -2.3254926204681396,
      "logps/chosen": -142.5237274169922,
      "logps/rejected": -150.29586791992188,
      "loss": 0.7873,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.08850278705358505,
      "rewards/margins": -0.12702789902687073,
      "rewards/rejected": 0.03852510452270508,
      "step": 356
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.8323207707310496e-05,
      "logits/chosen": -2.112971782684326,
      "logits/rejected": -2.1613588333129883,
      "logps/chosen": -170.9886932373047,
      "logps/rejected": -180.46363830566406,
      "loss": 0.68,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.04462575912475586,
      "rewards/margins": 0.08605735003948212,
      "rewards/rejected": -0.13068309426307678,
      "step": 357
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.831028265501764e-05,
      "logits/chosen": -1.8144406080245972,
      "logits/rejected": -1.8706724643707275,
      "logps/chosen": -172.77764892578125,
      "logps/rejected": -187.60598754882812,
      "loss": 0.7243,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.18522275984287262,
      "rewards/margins": 0.02290777862071991,
      "rewards/rejected": -0.20813053846359253,
      "step": 358
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.829730972140517e-05,
      "logits/chosen": -2.0918076038360596,
      "logits/rejected": -2.0828185081481934,
      "logps/chosen": -133.16778564453125,
      "logps/rejected": -139.5255889892578,
      "loss": 0.6419,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.05682823061943054,
      "rewards/margins": 0.1471494734287262,
      "rewards/rejected": -0.09032122790813446,
      "step": 359
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.8284288933120594e-05,
      "logits/chosen": -2.0551576614379883,
      "logits/rejected": -2.0017828941345215,
      "logps/chosen": -173.2173309326172,
      "logps/rejected": -187.19105529785156,
      "loss": 0.6964,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13149763643741608,
      "rewards/margins": 0.09333821386098862,
      "rewards/rejected": -0.2248358577489853,
      "step": 360
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.8271220316909735e-05,
      "logits/chosen": -2.1928532123565674,
      "logits/rejected": -2.184141159057617,
      "logps/chosen": -200.8461456298828,
      "logps/rejected": -204.56504821777344,
      "loss": 0.7748,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.022504782304167747,
      "rewards/margins": -0.08254070580005646,
      "rewards/rejected": 0.06003589183092117,
      "step": 361
    },
    {
      "epoch": 0.47,
      "learning_rate": 4.825810389961666e-05,
      "logits/chosen": -2.254242181777954,
      "logits/rejected": -2.261145830154419,
      "logps/chosen": -160.71571350097656,
      "logps/rejected": -142.36376953125,
      "loss": 0.6519,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.024668443948030472,
      "rewards/margins": 0.13744822144508362,
      "rewards/rejected": -0.16211667656898499,
      "step": 362
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.8244939708183596e-05,
      "logits/chosen": -2.1350438594818115,
      "logits/rejected": -2.1070897579193115,
      "logps/chosen": -160.7988739013672,
      "logps/rejected": -157.40176391601562,
      "loss": 0.7235,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.01498755719512701,
      "rewards/margins": -0.015977520495653152,
      "rewards/rejected": 0.03096509724855423,
      "step": 363
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.823172776965094e-05,
      "logits/chosen": -2.2463290691375732,
      "logits/rejected": -2.2258195877075195,
      "logps/chosen": -136.88392639160156,
      "logps/rejected": -128.96087646484375,
      "loss": 0.6831,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.055554769933223724,
      "rewards/margins": 0.06438815593719482,
      "rewards/rejected": -0.11994291841983795,
      "step": 364
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.821846811115713e-05,
      "logits/chosen": -1.9229159355163574,
      "logits/rejected": -1.839210867881775,
      "logps/chosen": -173.51405334472656,
      "logps/rejected": -153.61341857910156,
      "loss": 0.7675,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1366410106420517,
      "rewards/margins": -0.06360301375389099,
      "rewards/rejected": 0.20024403929710388,
      "step": 365
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.820516075993865e-05,
      "logits/chosen": -2.0157222747802734,
      "logits/rejected": -2.0540072917938232,
      "logps/chosen": -141.3604278564453,
      "logps/rejected": -146.56663513183594,
      "loss": 0.7431,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15194594860076904,
      "rewards/margins": 0.032469067722558975,
      "rewards/rejected": -0.1844150424003601,
      "step": 366
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.819180574332994e-05,
      "logits/chosen": -2.1887617111206055,
      "logits/rejected": -2.1101431846618652,
      "logps/chosen": -164.18893432617188,
      "logps/rejected": -160.06475830078125,
      "loss": 0.6856,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.02116868458688259,
      "rewards/margins": 0.028199315071105957,
      "rewards/rejected": -0.0493679977953434,
      "step": 367
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.8178403088763355e-05,
      "logits/chosen": -2.2744359970092773,
      "logits/rejected": -2.291210651397705,
      "logps/chosen": -168.8428955078125,
      "logps/rejected": -171.43356323242188,
      "loss": 0.8289,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.059717368334531784,
      "rewards/margins": -0.18386751413345337,
      "rewards/rejected": 0.12415014207363129,
      "step": 368
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.8164952823769085e-05,
      "logits/chosen": -1.8740739822387695,
      "logits/rejected": -1.8469749689102173,
      "logps/chosen": -135.71493530273438,
      "logps/rejected": -149.68069458007812,
      "loss": 0.6854,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.029844455420970917,
      "rewards/margins": 0.11711962521076202,
      "rewards/rejected": -0.14696410298347473,
      "step": 369
    },
    {
      "epoch": 0.48,
      "learning_rate": 4.815145497597514e-05,
      "logits/chosen": -2.102431297302246,
      "logits/rejected": -2.032944917678833,
      "logps/chosen": -173.45106506347656,
      "logps/rejected": -159.73878479003906,
      "loss": 0.9183,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.022377396002411842,
      "rewards/margins": -0.20098015666007996,
      "rewards/rejected": 0.22335757315158844,
      "step": 370
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.8137909573107246e-05,
      "logits/chosen": -2.3416833877563477,
      "logits/rejected": -2.3538575172424316,
      "logps/chosen": -167.69911193847656,
      "logps/rejected": -157.8156280517578,
      "loss": 0.7045,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.004629455506801605,
      "rewards/margins": 0.09061210602521896,
      "rewards/rejected": -0.08598263561725616,
      "step": 371
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.812431664298883e-05,
      "logits/chosen": -2.17607045173645,
      "logits/rejected": -2.1720361709594727,
      "logps/chosen": -166.7891845703125,
      "logps/rejected": -166.55804443359375,
      "loss": 0.7444,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.178691565990448,
      "rewards/margins": -0.029032886028289795,
      "rewards/rejected": -0.1496586799621582,
      "step": 372
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.811067621354094e-05,
      "logits/chosen": -2.0844247341156006,
      "logits/rejected": -2.150310516357422,
      "logps/chosen": -160.33216857910156,
      "logps/rejected": -183.65805053710938,
      "loss": 0.738,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.019554704427719116,
      "rewards/margins": -0.00408715195953846,
      "rewards/rejected": -0.015467546880245209,
      "step": 373
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.8096988312782174e-05,
      "logits/chosen": -2.148946762084961,
      "logits/rejected": -2.091033458709717,
      "logps/chosen": -163.8697052001953,
      "logps/rejected": -177.90403747558594,
      "loss": 0.7519,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.020494531840085983,
      "rewards/margins": -0.09285805374383926,
      "rewards/rejected": 0.11335259675979614,
      "step": 374
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.8083252968828665e-05,
      "logits/chosen": -2.034929037094116,
      "logits/rejected": -2.0719597339630127,
      "logps/chosen": -154.4242401123047,
      "logps/rejected": -157.4151611328125,
      "loss": 0.7685,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.11203078925609589,
      "rewards/margins": -0.1181333065032959,
      "rewards/rejected": 0.006102517247200012,
      "step": 375
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.8069470209893974e-05,
      "logits/chosen": -2.1990954875946045,
      "logits/rejected": -2.1845507621765137,
      "logps/chosen": -167.10202026367188,
      "logps/rejected": -173.89694213867188,
      "loss": 0.7338,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.048778437077999115,
      "rewards/margins": -0.013265417888760567,
      "rewards/rejected": -0.0355130136013031,
      "step": 376
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.8055640064289086e-05,
      "logits/chosen": -2.0750880241394043,
      "logits/rejected": -2.0607268810272217,
      "logps/chosen": -145.49095153808594,
      "logps/rejected": -146.54730224609375,
      "loss": 0.7064,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12515582144260406,
      "rewards/margins": 0.004622337408363819,
      "rewards/rejected": -0.1297781616449356,
      "step": 377
    },
    {
      "epoch": 0.49,
      "learning_rate": 4.80417625604223e-05,
      "logits/chosen": -2.065021276473999,
      "logits/rejected": -1.985723853111267,
      "logps/chosen": -168.58168029785156,
      "logps/rejected": -154.1855010986328,
      "loss": 0.7135,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.2825557291507721,
      "rewards/margins": -0.0009878575801849365,
      "rewards/rejected": 0.28354358673095703,
      "step": 378
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.8027837726799205e-05,
      "logits/chosen": -2.0231850147247314,
      "logits/rejected": -2.0219273567199707,
      "logps/chosen": -177.43508911132812,
      "logps/rejected": -180.77137756347656,
      "loss": 0.6861,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008570343255996704,
      "rewards/margins": 0.08320408314466476,
      "rewards/rejected": -0.07463373243808746,
      "step": 379
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.801386559202259e-05,
      "logits/chosen": -2.098315477371216,
      "logits/rejected": -2.0270769596099854,
      "logps/chosen": -174.91799926757812,
      "logps/rejected": -150.36114501953125,
      "loss": 0.7361,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.12160081416368484,
      "rewards/margins": 0.009557720273733139,
      "rewards/rejected": 0.112043097615242,
      "step": 380
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.799984618479242e-05,
      "logits/chosen": -1.9548274278640747,
      "logits/rejected": -1.9769983291625977,
      "logps/chosen": -142.3773193359375,
      "logps/rejected": -138.9167938232422,
      "loss": 0.8095,
      "rewards/accuracies": 0.125,
      "rewards/chosen": -0.1921553909778595,
      "rewards/margins": -0.17860561609268188,
      "rewards/rejected": -0.01354978233575821,
      "step": 381
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.798577953390577e-05,
      "logits/chosen": -2.0205070972442627,
      "logits/rejected": -2.027376413345337,
      "logps/chosen": -195.26025390625,
      "logps/rejected": -219.80625915527344,
      "loss": 0.7086,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27936846017837524,
      "rewards/margins": 0.01577301323413849,
      "rewards/rejected": -0.29514145851135254,
      "step": 382
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.797166566825675e-05,
      "logits/chosen": -2.071674346923828,
      "logits/rejected": -2.0323328971862793,
      "logps/chosen": -151.1609344482422,
      "logps/rejected": -169.61581420898438,
      "loss": 0.6817,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.14004096388816833,
      "rewards/margins": 0.09525477886199951,
      "rewards/rejected": 0.04478616267442703,
      "step": 383
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.795750461683644e-05,
      "logits/chosen": -2.1325645446777344,
      "logits/rejected": -2.059990882873535,
      "logps/chosen": -154.6941680908203,
      "logps/rejected": -157.33233642578125,
      "loss": 0.6819,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.07734552025794983,
      "rewards/margins": 0.0484672486782074,
      "rewards/rejected": 0.028878264129161835,
      "step": 384
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.794329640873285e-05,
      "logits/chosen": -2.094820737838745,
      "logits/rejected": -2.146373987197876,
      "logps/chosen": -145.1197509765625,
      "logps/rejected": -144.53829956054688,
      "loss": 0.5279,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18119190633296967,
      "rewards/margins": 0.4195486307144165,
      "rewards/rejected": -0.23835672438144684,
      "step": 385
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.7929041073130867e-05,
      "logits/chosen": -2.0487372875213623,
      "logits/rejected": -1.9535651206970215,
      "logps/chosen": -146.88800048828125,
      "logps/rejected": -146.015380859375,
      "loss": 0.5709,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05707988515496254,
      "rewards/margins": 0.39438337087631226,
      "rewards/rejected": -0.33730348944664,
      "step": 386
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.7914738639312165e-05,
      "logits/chosen": -1.9662736654281616,
      "logits/rejected": -2.0507540702819824,
      "logps/chosen": -159.54833984375,
      "logps/rejected": -181.677978515625,
      "loss": 0.8444,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.1504884660243988,
      "rewards/margins": -0.18733000755310059,
      "rewards/rejected": 0.036841537803411484,
      "step": 387
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.790038913665519e-05,
      "logits/chosen": -1.8254969120025635,
      "logits/rejected": -1.8466860055923462,
      "logps/chosen": -121.59519958496094,
      "logps/rejected": -131.51698303222656,
      "loss": 0.7607,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": 0.16594046354293823,
      "rewards/margins": -0.08683046698570251,
      "rewards/rejected": 0.25277090072631836,
      "step": 388
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.788599259463502e-05,
      "logits/chosen": -2.0490996837615967,
      "logits/rejected": -2.088806390762329,
      "logps/chosen": -144.5177459716797,
      "logps/rejected": -151.15444946289062,
      "loss": 0.7034,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.16910819709300995,
      "rewards/margins": 0.07333025336265564,
      "rewards/rejected": 0.09577794373035431,
      "step": 389
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.787154904282341e-05,
      "logits/chosen": -2.2665562629699707,
      "logits/rejected": -2.2620351314544678,
      "logps/chosen": -133.22760009765625,
      "logps/rejected": -147.43667602539062,
      "loss": 0.7657,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.18194647133350372,
      "rewards/margins": -0.09383340179920197,
      "rewards/rejected": -0.08811306953430176,
      "step": 390
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.7857058510888645e-05,
      "logits/chosen": -2.2096052169799805,
      "logits/rejected": -2.216580867767334,
      "logps/chosen": -155.67662048339844,
      "logps/rejected": -169.5216064453125,
      "loss": 0.7002,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0033222604542970657,
      "rewards/margins": 0.02931857667863369,
      "rewards/rejected": -0.02599630132317543,
      "step": 391
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.7842521028595526e-05,
      "logits/chosen": -2.1179494857788086,
      "logits/rejected": -2.172715187072754,
      "logps/chosen": -139.75497436523438,
      "logps/rejected": -149.2156982421875,
      "loss": 0.7134,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20826546847820282,
      "rewards/margins": 0.03352481871843338,
      "rewards/rejected": -0.2417902648448944,
      "step": 392
    },
    {
      "epoch": 0.51,
      "learning_rate": 4.7827936625805284e-05,
      "logits/chosen": -2.1453120708465576,
      "logits/rejected": -2.2276957035064697,
      "logps/chosen": -160.8773651123047,
      "logps/rejected": -182.9378204345703,
      "loss": 0.715,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.20616355538368225,
      "rewards/margins": 0.011941194534301758,
      "rewards/rejected": -0.21810473501682281,
      "step": 393
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.7813305332475535e-05,
      "logits/chosen": -1.5362883806228638,
      "logits/rejected": -1.4986250400543213,
      "logps/chosen": -256.8396301269531,
      "logps/rejected": -269.427001953125,
      "loss": 0.6733,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.024325193837285042,
      "rewards/margins": 0.11790871620178223,
      "rewards/rejected": -0.14223390817642212,
      "step": 394
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.77986271786602e-05,
      "logits/chosen": -2.1201870441436768,
      "logits/rejected": -2.150663137435913,
      "logps/chosen": -144.35189819335938,
      "logps/rejected": -152.51849365234375,
      "loss": 0.605,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.19814497232437134,
      "rewards/margins": 0.2016732394695282,
      "rewards/rejected": -0.39981821179389954,
      "step": 395
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.778390219450949e-05,
      "logits/chosen": -1.9574625492095947,
      "logits/rejected": -1.9414085149765015,
      "logps/chosen": -206.0396728515625,
      "logps/rejected": -230.67552185058594,
      "loss": 0.5939,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0024340637028217316,
      "rewards/margins": 0.2473735809326172,
      "rewards/rejected": -0.24493952095508575,
      "step": 396
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.776913041026976e-05,
      "logits/chosen": -2.2131030559539795,
      "logits/rejected": -2.1374146938323975,
      "logps/chosen": -157.73483276367188,
      "logps/rejected": -147.059326171875,
      "loss": 0.6997,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1409614086151123,
      "rewards/margins": 0.018417831510305405,
      "rewards/rejected": -0.15937921404838562,
      "step": 397
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.775431185628353e-05,
      "logits/chosen": -1.8067915439605713,
      "logits/rejected": -1.7414895296096802,
      "logps/chosen": -192.5210418701172,
      "logps/rejected": -151.17227172851562,
      "loss": 0.8156,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.41330477595329285,
      "rewards/margins": -0.11488999426364899,
      "rewards/rejected": -0.29841476678848267,
      "step": 398
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.7739446562989384e-05,
      "logits/chosen": -2.2105062007904053,
      "logits/rejected": -2.2545359134674072,
      "logps/chosen": -171.10385131835938,
      "logps/rejected": -176.2754364013672,
      "loss": 0.6499,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5258097648620605,
      "rewards/margins": 0.18083709478378296,
      "rewards/rejected": -0.7066469192504883,
      "step": 399
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.772453456092191e-05,
      "logits/chosen": -2.2050392627716064,
      "logits/rejected": -2.1882925033569336,
      "logps/chosen": -181.8564453125,
      "logps/rejected": -180.79759216308594,
      "loss": 0.7227,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.48925238847732544,
      "rewards/margins": -0.004873424768447876,
      "rewards/rejected": -0.4843789339065552,
      "step": 400
    },
    {
      "epoch": 0.52,
      "learning_rate": 4.7709575880711634e-05,
      "logits/chosen": -2.2256252765655518,
      "logits/rejected": -2.2812557220458984,
      "logps/chosen": -113.94125366210938,
      "logps/rejected": -123.2302474975586,
      "loss": 0.7755,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.23328812420368195,
      "rewards/margins": -0.1003262847661972,
      "rewards/rejected": -0.13296185433864594,
      "step": 401
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.769457055308497e-05,
      "logits/chosen": -2.215670108795166,
      "logits/rejected": -2.2150087356567383,
      "logps/chosen": -166.19100952148438,
      "logps/rejected": -174.61366271972656,
      "loss": 0.7388,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3156086504459381,
      "rewards/margins": -0.022609539330005646,
      "rewards/rejected": -0.29299911856651306,
      "step": 402
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.767951860886415e-05,
      "logits/chosen": -1.9413666725158691,
      "logits/rejected": -2.0344936847686768,
      "logps/chosen": -140.91476440429688,
      "logps/rejected": -164.15542602539062,
      "loss": 0.7518,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.37001854181289673,
      "rewards/margins": -0.07773025333881378,
      "rewards/rejected": -0.29228830337524414,
      "step": 403
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.766442007896715e-05,
      "logits/chosen": -2.185791254043579,
      "logits/rejected": -2.158402681350708,
      "logps/chosen": -174.4066925048828,
      "logps/rejected": -174.22225952148438,
      "loss": 0.6677,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.426675409078598,
      "rewards/margins": 0.14762084186077118,
      "rewards/rejected": -0.5742962956428528,
      "step": 404
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.764927499440767e-05,
      "logits/chosen": -1.982418417930603,
      "logits/rejected": -1.9350054264068604,
      "logps/chosen": -178.34976196289062,
      "logps/rejected": -180.20494079589844,
      "loss": 0.6274,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3919566869735718,
      "rewards/margins": 0.19768588244915009,
      "rewards/rejected": -0.5896425247192383,
      "step": 405
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.763408338629498e-05,
      "logits/chosen": -2.2213985919952393,
      "logits/rejected": -2.2709920406341553,
      "logps/chosen": -143.23550415039062,
      "logps/rejected": -147.50332641601562,
      "loss": 0.6232,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.20936697721481323,
      "rewards/margins": 0.1896706074476242,
      "rewards/rejected": -0.39903756976127625,
      "step": 406
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.761884528583396e-05,
      "logits/chosen": -2.2714898586273193,
      "logits/rejected": -2.1939728260040283,
      "logps/chosen": -182.6309356689453,
      "logps/rejected": -193.048583984375,
      "loss": 0.7323,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4604118764400482,
      "rewards/margins": 0.03731643036007881,
      "rewards/rejected": -0.49772825837135315,
      "step": 407
    },
    {
      "epoch": 0.53,
      "learning_rate": 4.760356072432498e-05,
      "logits/chosen": -2.160900115966797,
      "logits/rejected": -2.0733065605163574,
      "logps/chosen": -184.3874053955078,
      "logps/rejected": -181.5076904296875,
      "loss": 0.6753,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.27709901332855225,
      "rewards/margins": 0.06253170967102051,
      "rewards/rejected": -0.33963072299957275,
      "step": 408
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.7588229733163834e-05,
      "logits/chosen": -1.7874306440353394,
      "logits/rejected": -1.8620976209640503,
      "logps/chosen": -207.32083129882812,
      "logps/rejected": -205.32540893554688,
      "loss": 0.6809,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5504447817802429,
      "rewards/margins": 0.07099004089832306,
      "rewards/rejected": -0.6214348077774048,
      "step": 409
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.757285234384169e-05,
      "logits/chosen": -2.2311081886291504,
      "logits/rejected": -2.1636242866516113,
      "logps/chosen": -165.6267852783203,
      "logps/rejected": -168.37962341308594,
      "loss": 0.7484,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.8211495280265808,
      "rewards/margins": -0.0740201398730278,
      "rewards/rejected": -0.7471294403076172,
      "step": 410
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.755742858794503e-05,
      "logits/chosen": -2.26678729057312,
      "logits/rejected": -2.2344679832458496,
      "logps/chosen": -157.74423217773438,
      "logps/rejected": -152.1922149658203,
      "loss": 0.7029,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.18589326739311218,
      "rewards/margins": 0.01804957166314125,
      "rewards/rejected": -0.20394286513328552,
      "step": 411
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.754195849715557e-05,
      "logits/chosen": -2.143720865249634,
      "logits/rejected": -2.176121711730957,
      "logps/chosen": -171.01498413085938,
      "logps/rejected": -174.25668334960938,
      "loss": 0.7835,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.34361717104911804,
      "rewards/margins": -0.003480616956949234,
      "rewards/rejected": -0.3401365578174591,
      "step": 412
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.75264421032502e-05,
      "logits/chosen": -2.2774815559387207,
      "logits/rejected": -2.2418551445007324,
      "logps/chosen": -183.46746826171875,
      "logps/rejected": -172.81829833984375,
      "loss": 0.6842,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.40849414467811584,
      "rewards/margins": 0.09351891279220581,
      "rewards/rejected": -0.5020129680633545,
      "step": 413
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.751087943810093e-05,
      "logits/chosen": -1.9202462434768677,
      "logits/rejected": -1.9200413227081299,
      "logps/chosen": -178.16827392578125,
      "logps/rejected": -163.49049377441406,
      "loss": 0.7876,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.6583839058876038,
      "rewards/margins": -0.0725487470626831,
      "rewards/rejected": -0.5858351588249207,
      "step": 414
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.749527053367481e-05,
      "logits/chosen": -2.3440024852752686,
      "logits/rejected": -2.306640386581421,
      "logps/chosen": -204.37374877929688,
      "logps/rejected": -196.37139892578125,
      "loss": 0.7817,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3725470006465912,
      "rewards/margins": -0.10262566059827805,
      "rewards/rejected": -0.26992136240005493,
      "step": 415
    },
    {
      "epoch": 0.54,
      "learning_rate": 4.747961542203386e-05,
      "logits/chosen": -2.206644296646118,
      "logits/rejected": -2.216529607772827,
      "logps/chosen": -145.55783081054688,
      "logps/rejected": -149.01528930664062,
      "loss": 0.651,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.29852521419525146,
      "rewards/margins": 0.12249046564102173,
      "rewards/rejected": -0.42101573944091797,
      "step": 416
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.746391413533503e-05,
      "logits/chosen": -1.9940528869628906,
      "logits/rejected": -1.959697961807251,
      "logps/chosen": -162.80947875976562,
      "logps/rejected": -181.39344787597656,
      "loss": 0.6996,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5685576796531677,
      "rewards/margins": 0.06293636560440063,
      "rewards/rejected": -0.6314940452575684,
      "step": 417
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.74481667058301e-05,
      "logits/chosen": -2.064054250717163,
      "logits/rejected": -2.1215896606445312,
      "logps/chosen": -154.61865234375,
      "logps/rejected": -155.92892456054688,
      "loss": 0.6675,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4008215069770813,
      "rewards/margins": 0.12303955852985382,
      "rewards/rejected": -0.5238610506057739,
      "step": 418
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.743237316586564e-05,
      "logits/chosen": -2.2184903621673584,
      "logits/rejected": -2.3070037364959717,
      "logps/chosen": -149.61441040039062,
      "logps/rejected": -151.4566650390625,
      "loss": 0.7559,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3738159239292145,
      "rewards/margins": -0.02443253993988037,
      "rewards/rejected": -0.3493833541870117,
      "step": 419
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.741653354788295e-05,
      "logits/chosen": -2.168128252029419,
      "logits/rejected": -2.1225526332855225,
      "logps/chosen": -160.09083557128906,
      "logps/rejected": -169.74386596679688,
      "loss": 0.7091,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5666425228118896,
      "rewards/margins": 0.06661619246006012,
      "rewards/rejected": -0.6332587003707886,
      "step": 420
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.7400647884417956e-05,
      "logits/chosen": -1.9577604532241821,
      "logits/rejected": -1.9241951704025269,
      "logps/chosen": -207.59442138671875,
      "logps/rejected": -217.36038208007812,
      "loss": 0.6921,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.37186557054519653,
      "rewards/margins": 0.10641665011644363,
      "rewards/rejected": -0.47828227281570435,
      "step": 421
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.7384716208101166e-05,
      "logits/chosen": -2.064387559890747,
      "logits/rejected": -2.022218942642212,
      "logps/chosen": -189.0228271484375,
      "logps/rejected": -185.25689697265625,
      "loss": 0.6958,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4377724528312683,
      "rewards/margins": 0.09676932543516159,
      "rewards/rejected": -0.5345417857170105,
      "step": 422
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.736873855165762e-05,
      "logits/chosen": -2.2576003074645996,
      "logits/rejected": -2.309368848800659,
      "logps/chosen": -180.17724609375,
      "logps/rejected": -186.60797119140625,
      "loss": 0.7273,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.19769221544265747,
      "rewards/margins": 0.021774642169475555,
      "rewards/rejected": -0.21946686506271362,
      "step": 423
    },
    {
      "epoch": 0.55,
      "learning_rate": 4.735271494790678e-05,
      "logits/chosen": -2.0674479007720947,
      "logits/rejected": -2.043943166732788,
      "logps/chosen": -189.95338439941406,
      "logps/rejected": -188.52682495117188,
      "loss": 0.8214,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.30729472637176514,
      "rewards/margins": -0.16690713167190552,
      "rewards/rejected": -0.14038759469985962,
      "step": 424
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.733664542976253e-05,
      "logits/chosen": -2.2818522453308105,
      "logits/rejected": -2.243852138519287,
      "logps/chosen": -139.22390747070312,
      "logps/rejected": -138.7141876220703,
      "loss": 0.6405,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1333819478750229,
      "rewards/margins": 0.24985717236995697,
      "rewards/rejected": -0.38323909044265747,
      "step": 425
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.732053003023301e-05,
      "logits/chosen": -2.3318722248077393,
      "logits/rejected": -2.331906795501709,
      "logps/chosen": -170.11148071289062,
      "logps/rejected": -154.28684997558594,
      "loss": 0.8989,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.4540347456932068,
      "rewards/margins": -0.20150336623191833,
      "rewards/rejected": -0.25253134965896606,
      "step": 426
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.730436878242064e-05,
      "logits/chosen": -2.0526726245880127,
      "logits/rejected": -2.094531536102295,
      "logps/chosen": -148.9898681640625,
      "logps/rejected": -150.51864624023438,
      "loss": 0.6562,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.34212395548820496,
      "rewards/margins": 0.1897253841161728,
      "rewards/rejected": -0.5318493247032166,
      "step": 427
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.7288161719522016e-05,
      "logits/chosen": -2.161623954772949,
      "logits/rejected": -2.1211910247802734,
      "logps/chosen": -151.83523559570312,
      "logps/rejected": -140.38848876953125,
      "loss": 0.7175,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.545448899269104,
      "rewards/margins": 0.03246283903717995,
      "rewards/rejected": -0.5779117345809937,
      "step": 428
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.727190887482783e-05,
      "logits/chosen": -2.1172046661376953,
      "logits/rejected": -2.05790376663208,
      "logps/chosen": -141.56634521484375,
      "logps/rejected": -152.59849548339844,
      "loss": 0.7776,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.29310688376426697,
      "rewards/margins": 0.007600661367177963,
      "rewards/rejected": -0.30070751905441284,
      "step": 429
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.725561028172282e-05,
      "logits/chosen": -2.2756776809692383,
      "logits/rejected": -2.261565685272217,
      "logps/chosen": -121.21385955810547,
      "logps/rejected": -120.08661651611328,
      "loss": 0.6775,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.21489115059375763,
      "rewards/margins": 0.12781137228012085,
      "rewards/rejected": -0.3427025377750397,
      "step": 430
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.7239265973685696e-05,
      "logits/chosen": -2.1066219806671143,
      "logits/rejected": -2.201887845993042,
      "logps/chosen": -129.29530334472656,
      "logps/rejected": -147.51333618164062,
      "loss": 0.7801,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6134562492370605,
      "rewards/margins": -0.030932441353797913,
      "rewards/rejected": -0.5825238227844238,
      "step": 431
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.722287598428907e-05,
      "logits/chosen": -2.0897793769836426,
      "logits/rejected": -2.091312885284424,
      "logps/chosen": -150.09129333496094,
      "logps/rejected": -157.86312866210938,
      "loss": 0.7574,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5798418521881104,
      "rewards/margins": -0.0995514839887619,
      "rewards/rejected": -0.48029035329818726,
      "step": 432
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.720644034719938e-05,
      "logits/chosen": -2.1701114177703857,
      "logits/rejected": -2.149559259414673,
      "logps/chosen": -178.9230499267578,
      "logps/rejected": -163.4266815185547,
      "loss": 0.7959,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5811534523963928,
      "rewards/margins": -0.08480577170848846,
      "rewards/rejected": -0.49634772539138794,
      "step": 433
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.7189959096176825e-05,
      "logits/chosen": -2.404374837875366,
      "logits/rejected": -2.407968521118164,
      "logps/chosen": -149.89866638183594,
      "logps/rejected": -149.10494995117188,
      "loss": 0.7978,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.4539230465888977,
      "rewards/margins": -0.16405440866947174,
      "rewards/rejected": -0.2898685932159424,
      "step": 434
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.7173432265075334e-05,
      "logits/chosen": -2.3623311519622803,
      "logits/rejected": -2.344980239868164,
      "logps/chosen": -233.85638427734375,
      "logps/rejected": -233.85716247558594,
      "loss": 0.7299,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4674505591392517,
      "rewards/margins": 0.010146500542759895,
      "rewards/rejected": -0.47759705781936646,
      "step": 435
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.7156859887842416e-05,
      "logits/chosen": -2.444455623626709,
      "logits/rejected": -2.4147419929504395,
      "logps/chosen": -157.481201171875,
      "logps/rejected": -155.94912719726562,
      "loss": 0.8272,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4809521436691284,
      "rewards/margins": -0.1707230508327484,
      "rewards/rejected": -0.3102290630340576,
      "step": 436
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.714024199851915e-05,
      "logits/chosen": -2.13613224029541,
      "logits/rejected": -2.11737060546875,
      "logps/chosen": -196.46791076660156,
      "logps/rejected": -206.36705017089844,
      "loss": 0.6643,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.41040414571762085,
      "rewards/margins": 0.20290464162826538,
      "rewards/rejected": -0.613308846950531,
      "step": 437
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.712357863124013e-05,
      "logits/chosen": -2.302917003631592,
      "logits/rejected": -2.30071759223938,
      "logps/chosen": -153.2189483642578,
      "logps/rejected": -148.35671997070312,
      "loss": 0.8639,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5300933122634888,
      "rewards/margins": -0.23618030548095703,
      "rewards/rejected": -0.29391294717788696,
      "step": 438
    },
    {
      "epoch": 0.57,
      "learning_rate": 4.710686982023332e-05,
      "logits/chosen": -2.1037135124206543,
      "logits/rejected": -2.084882974624634,
      "logps/chosen": -171.69317626953125,
      "logps/rejected": -184.76455688476562,
      "loss": 0.8317,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.7027316093444824,
      "rewards/margins": -0.20845447480678558,
      "rewards/rejected": -0.49427708983421326,
      "step": 439
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.709011559982006e-05,
      "logits/chosen": -2.315164566040039,
      "logits/rejected": -2.35860538482666,
      "logps/chosen": -142.0930938720703,
      "logps/rejected": -144.05953979492188,
      "loss": 0.538,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2587888836860657,
      "rewards/margins": 0.43140506744384766,
      "rewards/rejected": -0.6901938915252686,
      "step": 440
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.707331600441495e-05,
      "logits/chosen": -2.025707483291626,
      "logits/rejected": -2.1452269554138184,
      "logps/chosen": -135.1129608154297,
      "logps/rejected": -180.877685546875,
      "loss": 0.7269,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48439598083496094,
      "rewards/margins": 0.12395285069942474,
      "rewards/rejected": -0.6083488464355469,
      "step": 441
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.705647106852581e-05,
      "logits/chosen": -2.1537985801696777,
      "logits/rejected": -2.1138670444488525,
      "logps/chosen": -161.801025390625,
      "logps/rejected": -152.15338134765625,
      "loss": 0.6527,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5599794983863831,
      "rewards/margins": 0.14886192977428436,
      "rewards/rejected": -0.7088414430618286,
      "step": 442
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.7039580826753564e-05,
      "logits/chosen": -1.9758260250091553,
      "logits/rejected": -2.015881299972534,
      "logps/chosen": -184.20703125,
      "logps/rejected": -191.9273681640625,
      "loss": 0.6208,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.572418749332428,
      "rewards/margins": 0.2520812749862671,
      "rewards/rejected": -0.8244999647140503,
      "step": 443
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.7022645313792235e-05,
      "logits/chosen": -2.2235710620880127,
      "logits/rejected": -2.2551536560058594,
      "logps/chosen": -152.44741821289062,
      "logps/rejected": -160.27525329589844,
      "loss": 0.792,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.53495192527771,
      "rewards/margins": -0.10131815075874329,
      "rewards/rejected": -0.4336337447166443,
      "step": 444
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.700566456442882e-05,
      "logits/chosen": -2.0654869079589844,
      "logits/rejected": -2.1601076126098633,
      "logps/chosen": -129.15142822265625,
      "logps/rejected": -140.05062866210938,
      "loss": 0.7985,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.4286402463912964,
      "rewards/margins": -0.08634166419506073,
      "rewards/rejected": -0.3422985374927521,
      "step": 445
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.6988638613543216e-05,
      "logits/chosen": -2.106463670730591,
      "logits/rejected": -2.1899609565734863,
      "logps/chosen": -207.76812744140625,
      "logps/rejected": -217.55172729492188,
      "loss": 0.7031,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.49319323897361755,
      "rewards/margins": 0.05841163173317909,
      "rewards/rejected": -0.5516048669815063,
      "step": 446
    },
    {
      "epoch": 0.58,
      "learning_rate": 4.6971567496108206e-05,
      "logits/chosen": -2.287020683288574,
      "logits/rejected": -2.302515745162964,
      "logps/chosen": -146.24276733398438,
      "logps/rejected": -157.62596130371094,
      "loss": 0.7239,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.35381338000297546,
      "rewards/margins": -0.004093277268111706,
      "rewards/rejected": -0.3497200906276703,
      "step": 447
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.695445124718931e-05,
      "logits/chosen": -2.316082000732422,
      "logits/rejected": -2.2804858684539795,
      "logps/chosen": -172.0008087158203,
      "logps/rejected": -177.33901977539062,
      "loss": 0.7188,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4350382685661316,
      "rewards/margins": -0.01686153933405876,
      "rewards/rejected": -0.41817668080329895,
      "step": 448
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.693728990194479e-05,
      "logits/chosen": -2.356797695159912,
      "logits/rejected": -2.373162269592285,
      "logps/chosen": -187.43911743164062,
      "logps/rejected": -201.7132568359375,
      "loss": 0.6999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3604944050312042,
      "rewards/margins": 0.06350287795066833,
      "rewards/rejected": -0.42399728298187256,
      "step": 449
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.692008349562551e-05,
      "logits/chosen": -2.3371644020080566,
      "logits/rejected": -2.244069814682007,
      "logps/chosen": -157.56373596191406,
      "logps/rejected": -158.41690063476562,
      "loss": 0.7745,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.7513232827186584,
      "rewards/margins": -0.11284235119819641,
      "rewards/rejected": -0.6384809613227844,
      "step": 450
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.690283206357491e-05,
      "logits/chosen": -1.9031703472137451,
      "logits/rejected": -1.8866480588912964,
      "logps/chosen": -137.91941833496094,
      "logps/rejected": -149.28627014160156,
      "loss": 0.6434,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.44801104068756104,
      "rewards/margins": 0.23882272839546204,
      "rewards/rejected": -0.6868337392807007,
      "step": 451
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.6885535641228904e-05,
      "logits/chosen": -2.1832618713378906,
      "logits/rejected": -2.121121406555176,
      "logps/chosen": -158.70449829101562,
      "logps/rejected": -139.44195556640625,
      "loss": 0.7216,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.542538046836853,
      "rewards/margins": -0.03218982741236687,
      "rewards/rejected": -0.5103481411933899,
      "step": 452
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.6868194264115833e-05,
      "logits/chosen": -2.2379095554351807,
      "logits/rejected": -2.124793291091919,
      "logps/chosen": -171.65277099609375,
      "logps/rejected": -158.50015258789062,
      "loss": 0.7441,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5649640560150146,
      "rewards/margins": -0.034692008048295975,
      "rewards/rejected": -0.5302720665931702,
      "step": 453
    },
    {
      "epoch": 0.59,
      "learning_rate": 4.685080796785637e-05,
      "logits/chosen": -2.2778732776641846,
      "logits/rejected": -2.2594337463378906,
      "logps/chosen": -151.48411560058594,
      "logps/rejected": -149.89930725097656,
      "loss": 0.6605,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.48167526721954346,
      "rewards/margins": 0.17855703830718994,
      "rewards/rejected": -0.6602323651313782,
      "step": 454
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.683337678816345e-05,
      "logits/chosen": -2.148597478866577,
      "logits/rejected": -2.1484172344207764,
      "logps/chosen": -143.36769104003906,
      "logps/rejected": -158.61419677734375,
      "loss": 0.6751,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5770382881164551,
      "rewards/margins": 0.10789009928703308,
      "rewards/rejected": -0.6849284172058105,
      "step": 455
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.6815900760842236e-05,
      "logits/chosen": -2.4137473106384277,
      "logits/rejected": -2.4021241664886475,
      "logps/chosen": -154.04502868652344,
      "logps/rejected": -157.95596313476562,
      "loss": 0.5879,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.4364089369773865,
      "rewards/margins": 0.2789525091648102,
      "rewards/rejected": -0.715361475944519,
      "step": 456
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.679837992178996e-05,
      "logits/chosen": -2.3135428428649902,
      "logits/rejected": -2.2878835201263428,
      "logps/chosen": -181.33885192871094,
      "logps/rejected": -168.310546875,
      "loss": 0.8835,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.7887184619903564,
      "rewards/margins": -0.2950814366340637,
      "rewards/rejected": -0.4936370253562927,
      "step": 457
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.678081430699594e-05,
      "logits/chosen": -2.263319492340088,
      "logits/rejected": -2.260213613510132,
      "logps/chosen": -159.97354125976562,
      "logps/rejected": -173.87554931640625,
      "loss": 0.7764,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6154264211654663,
      "rewards/margins": 0.04768490046262741,
      "rewards/rejected": -0.6631112694740295,
      "step": 458
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.676320395254146e-05,
      "logits/chosen": -2.1689982414245605,
      "logits/rejected": -2.0917975902557373,
      "logps/chosen": -199.61489868164062,
      "logps/rejected": -203.17724609375,
      "loss": 0.684,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5080626606941223,
      "rewards/margins": 0.14702026546001434,
      "rewards/rejected": -0.6550828218460083,
      "step": 459
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.674554889459968e-05,
      "logits/chosen": -2.080754518508911,
      "logits/rejected": -2.030247449874878,
      "logps/chosen": -142.29052734375,
      "logps/rejected": -165.71142578125,
      "loss": 0.7408,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3904760479927063,
      "rewards/margins": 0.15203596651554108,
      "rewards/rejected": -0.542512059211731,
      "step": 460
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.672784916943562e-05,
      "logits/chosen": -1.9848957061767578,
      "logits/rejected": -1.9429978132247925,
      "logps/chosen": -186.40225219726562,
      "logps/rejected": -179.22129821777344,
      "loss": 0.7617,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.8649554252624512,
      "rewards/margins": -0.06034373492002487,
      "rewards/rejected": -0.8046116828918457,
      "step": 461
    },
    {
      "epoch": 0.6,
      "learning_rate": 4.6710104813406034e-05,
      "logits/chosen": -2.0926647186279297,
      "logits/rejected": -2.121751070022583,
      "logps/chosen": -170.80023193359375,
      "logps/rejected": -175.02835083007812,
      "loss": 0.7499,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.61098712682724,
      "rewards/margins": 0.014241974800825119,
      "rewards/rejected": -0.6252290606498718,
      "step": 462
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.669231586295934e-05,
      "logits/chosen": -1.9556676149368286,
      "logits/rejected": -1.9728052616119385,
      "logps/chosen": -213.68606567382812,
      "logps/rejected": -174.6243896484375,
      "loss": 0.8219,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.6639366149902344,
      "rewards/margins": -0.16740846633911133,
      "rewards/rejected": -0.49652814865112305,
      "step": 463
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.667448235463557e-05,
      "logits/chosen": -2.060239553451538,
      "logits/rejected": -2.061102867126465,
      "logps/chosen": -137.50906372070312,
      "logps/rejected": -128.2222137451172,
      "loss": 0.818,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.2569347321987152,
      "rewards/margins": -0.15142616629600525,
      "rewards/rejected": -0.10550854355096817,
      "step": 464
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.665660432506629e-05,
      "logits/chosen": -2.1114845275878906,
      "logits/rejected": -2.12737774848938,
      "logps/chosen": -169.5026397705078,
      "logps/rejected": -141.35501098632812,
      "loss": 0.7919,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.30224573612213135,
      "rewards/margins": -0.09046731889247894,
      "rewards/rejected": -0.2117784172296524,
      "step": 465
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.6638681810974496e-05,
      "logits/chosen": -2.0212507247924805,
      "logits/rejected": -2.1288084983825684,
      "logps/chosen": -147.18545532226562,
      "logps/rejected": -166.57452392578125,
      "loss": 0.7137,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.31484949588775635,
      "rewards/margins": 0.05684095248579979,
      "rewards/rejected": -0.37169045209884644,
      "step": 466
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.6620714849174576e-05,
      "logits/chosen": -2.2373719215393066,
      "logits/rejected": -2.2259066104888916,
      "logps/chosen": -221.53643798828125,
      "logps/rejected": -209.2789306640625,
      "loss": 0.6621,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.18473269045352936,
      "rewards/margins": 0.11960664391517639,
      "rewards/rejected": -0.30433934926986694,
      "step": 467
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.660270347657219e-05,
      "logits/chosen": -2.235313892364502,
      "logits/rejected": -2.207859992980957,
      "logps/chosen": -145.51107788085938,
      "logps/rejected": -132.99411010742188,
      "loss": 0.8067,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4317854642868042,
      "rewards/margins": -0.16359470784664154,
      "rewards/rejected": -0.26819074153900146,
      "step": 468
    },
    {
      "epoch": 0.61,
      "learning_rate": 4.658464773016428e-05,
      "logits/chosen": -2.1742892265319824,
      "logits/rejected": -2.266177177429199,
      "logps/chosen": -142.66419982910156,
      "logps/rejected": -159.65928649902344,
      "loss": 0.6783,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.25551578402519226,
      "rewards/margins": 0.11744444072246552,
      "rewards/rejected": -0.3729602098464966,
      "step": 469
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.6566547647038864e-05,
      "logits/chosen": -2.1665616035461426,
      "logits/rejected": -2.1410114765167236,
      "logps/chosen": -172.94570922851562,
      "logps/rejected": -167.9188995361328,
      "loss": 0.7223,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.34934595227241516,
      "rewards/margins": 0.025624670088291168,
      "rewards/rejected": -0.37497058510780334,
      "step": 470
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.6548403264375074e-05,
      "logits/chosen": -2.055755615234375,
      "logits/rejected": -2.020624876022339,
      "logps/chosen": -150.6020965576172,
      "logps/rejected": -185.5800323486328,
      "loss": 0.6847,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5740702748298645,
      "rewards/margins": 0.09851770102977753,
      "rewards/rejected": -0.6725879907608032,
      "step": 471
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.6530214619443037e-05,
      "logits/chosen": -2.0328664779663086,
      "logits/rejected": -2.0425925254821777,
      "logps/chosen": -167.3757781982422,
      "logps/rejected": -173.4803466796875,
      "loss": 0.715,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4150983989238739,
      "rewards/margins": 0.005986988544464111,
      "rewards/rejected": -0.421085387468338,
      "step": 472
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.6511981749603775e-05,
      "logits/chosen": -1.9052200317382812,
      "logits/rejected": -1.998968243598938,
      "logps/chosen": -123.46515655517578,
      "logps/rejected": -141.20614624023438,
      "loss": 0.6627,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01827160082757473,
      "rewards/margins": 0.1650063842535019,
      "rewards/rejected": -0.14673477411270142,
      "step": 473
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.6493704692309175e-05,
      "logits/chosen": -2.1721608638763428,
      "logits/rejected": -2.181016445159912,
      "logps/chosen": -169.97547912597656,
      "logps/rejected": -174.20501708984375,
      "loss": 0.6978,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.34234559535980225,
      "rewards/margins": 0.039180606603622437,
      "rewards/rejected": -0.3815262019634247,
      "step": 474
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.647538348510189e-05,
      "logits/chosen": -2.042267084121704,
      "logits/rejected": -2.0465734004974365,
      "logps/chosen": -141.28273010253906,
      "logps/rejected": -137.6270294189453,
      "loss": 0.7715,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.45353037118911743,
      "rewards/margins": -0.10618551820516586,
      "rewards/rejected": -0.34734484553337097,
      "step": 475
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.645701816561523e-05,
      "logits/chosen": -2.080418825149536,
      "logits/rejected": -2.0591931343078613,
      "logps/chosen": -156.5729217529297,
      "logps/rejected": -160.69155883789062,
      "loss": 0.8914,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.3119348883628845,
      "rewards/margins": -0.33930426836013794,
      "rewards/rejected": 0.027369357645511627,
      "step": 476
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.643860877157314e-05,
      "logits/chosen": -1.639522671699524,
      "logits/rejected": -1.6502141952514648,
      "logps/chosen": -178.24334716796875,
      "logps/rejected": -169.12493896484375,
      "loss": 0.7049,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.44444751739501953,
      "rewards/margins": 0.014572631567716599,
      "rewards/rejected": -0.45902013778686523,
      "step": 477
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.642015534079012e-05,
      "logits/chosen": -2.047992467880249,
      "logits/rejected": -2.0363173484802246,
      "logps/chosen": -134.27391052246094,
      "logps/rejected": -123.03015899658203,
      "loss": 0.7044,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.21198835968971252,
      "rewards/margins": 0.025383614003658295,
      "rewards/rejected": -0.23737198114395142,
      "step": 478
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.640165791117106e-05,
      "logits/chosen": -2.0618531703948975,
      "logits/rejected": -2.0514118671417236,
      "logps/chosen": -152.69686889648438,
      "logps/rejected": -135.8447265625,
      "loss": 0.6722,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.04251031577587128,
      "rewards/margins": 0.09286715090274811,
      "rewards/rejected": -0.13537748157978058,
      "step": 479
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.63831165207113e-05,
      "logits/chosen": -2.015552282333374,
      "logits/rejected": -1.9658578634262085,
      "logps/chosen": -143.47520446777344,
      "logps/rejected": -146.58863830566406,
      "loss": 0.8025,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.14033322036266327,
      "rewards/margins": -0.1578764021396637,
      "rewards/rejected": 0.01754317432641983,
      "step": 480
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.6364531207496426e-05,
      "logits/chosen": -2.1612448692321777,
      "logits/rejected": -2.099830389022827,
      "logps/chosen": -161.23220825195312,
      "logps/rejected": -143.65408325195312,
      "loss": 0.7213,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.29882240295410156,
      "rewards/margins": -0.008122054859995842,
      "rewards/rejected": -0.2907003164291382,
      "step": 481
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.634590200970227e-05,
      "logits/chosen": -1.8700077533721924,
      "logits/rejected": -1.9340232610702515,
      "logps/chosen": -160.71038818359375,
      "logps/rejected": -151.01939392089844,
      "loss": 0.7197,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3910747170448303,
      "rewards/margins": 0.08369296789169312,
      "rewards/rejected": -0.47476768493652344,
      "step": 482
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.632722896559481e-05,
      "logits/chosen": -2.1195974349975586,
      "logits/rejected": -2.1336312294006348,
      "logps/chosen": -151.23971557617188,
      "logps/rejected": -160.65672302246094,
      "loss": 0.78,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.08127432316541672,
      "rewards/margins": -0.11872512847185135,
      "rewards/rejected": 0.19999945163726807,
      "step": 483
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.630851211353007e-05,
      "logits/chosen": -2.0612592697143555,
      "logits/rejected": -2.0779013633728027,
      "logps/chosen": -131.81069946289062,
      "logps/rejected": -129.8345489501953,
      "loss": 0.689,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03394623100757599,
      "rewards/margins": 0.08058298379182816,
      "rewards/rejected": -0.11452920734882355,
      "step": 484
    },
    {
      "epoch": 0.63,
      "learning_rate": 4.628975149195407e-05,
      "logits/chosen": -1.9617186784744263,
      "logits/rejected": -1.9468779563903809,
      "logps/chosen": -130.2001495361328,
      "logps/rejected": -125.31912994384766,
      "loss": 0.8841,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.41198205947875977,
      "rewards/margins": -0.24151001870632172,
      "rewards/rejected": -0.17047205567359924,
      "step": 485
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.6270947139402744e-05,
      "logits/chosen": -2.0373647212982178,
      "logits/rejected": -2.0272598266601562,
      "logps/chosen": -161.21817016601562,
      "logps/rejected": -160.80905151367188,
      "loss": 0.5759,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23811860382556915,
      "rewards/margins": 0.3550880253314972,
      "rewards/rejected": -0.5932066440582275,
      "step": 486
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.6252099094501834e-05,
      "logits/chosen": -2.263495922088623,
      "logits/rejected": -2.220659017562866,
      "logps/chosen": -150.18930053710938,
      "logps/rejected": -147.73287963867188,
      "loss": 0.6553,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.039855338633060455,
      "rewards/margins": 0.12497195601463318,
      "rewards/rejected": -0.16482731699943542,
      "step": 487
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.623320739596685e-05,
      "logits/chosen": -2.1021156311035156,
      "logits/rejected": -2.120116949081421,
      "logps/chosen": -153.40098571777344,
      "logps/rejected": -157.4463653564453,
      "loss": 0.7669,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09802110493183136,
      "rewards/margins": -0.008432537317276001,
      "rewards/rejected": -0.08958857506513596,
      "step": 488
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.621427208260296e-05,
      "logits/chosen": -1.9562160968780518,
      "logits/rejected": -2.006755828857422,
      "logps/chosen": -165.9380340576172,
      "logps/rejected": -173.2657928466797,
      "loss": 0.7826,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6031227111816406,
      "rewards/margins": -0.10150633007287979,
      "rewards/rejected": -0.501616358757019,
      "step": 489
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.6195293193304915e-05,
      "logits/chosen": -1.9626574516296387,
      "logits/rejected": -1.9933884143829346,
      "logps/chosen": -187.50564575195312,
      "logps/rejected": -184.9365234375,
      "loss": 0.7809,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.05860882252454758,
      "rewards/margins": -0.11062480509281158,
      "rewards/rejected": 0.05201598256826401,
      "step": 490
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.6176270767056976e-05,
      "logits/chosen": -2.0054826736450195,
      "logits/rejected": -2.002007484436035,
      "logps/chosen": -172.8408966064453,
      "logps/rejected": -165.1022491455078,
      "loss": 0.7222,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1988084763288498,
      "rewards/margins": -0.013871286064386368,
      "rewards/rejected": -0.18493719398975372,
      "step": 491
    },
    {
      "epoch": 0.64,
      "learning_rate": 4.615720484293286e-05,
      "logits/chosen": -2.0947346687316895,
      "logits/rejected": -2.138517379760742,
      "logps/chosen": -148.93450927734375,
      "logps/rejected": -144.9428253173828,
      "loss": 0.6561,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12516680359840393,
      "rewards/margins": 0.15067778527736664,
      "rewards/rejected": -0.2758445739746094,
      "step": 492
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.613809546009558e-05,
      "logits/chosen": -2.115553140640259,
      "logits/rejected": -2.149724006652832,
      "logps/chosen": -153.2530059814453,
      "logps/rejected": -160.34811401367188,
      "loss": 0.6297,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14684545993804932,
      "rewards/margins": 0.21833759546279907,
      "rewards/rejected": -0.365183025598526,
      "step": 493
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.611894265779748e-05,
      "logits/chosen": -2.068711280822754,
      "logits/rejected": -2.126939058303833,
      "logps/chosen": -139.7198486328125,
      "logps/rejected": -151.35397338867188,
      "loss": 0.6535,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.08141559362411499,
      "rewards/margins": 0.1623344123363495,
      "rewards/rejected": -0.24375002086162567,
      "step": 494
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.609974647538003e-05,
      "logits/chosen": -2.225661516189575,
      "logits/rejected": -2.150395393371582,
      "logps/chosen": -132.438232421875,
      "logps/rejected": -128.7378692626953,
      "loss": 0.7277,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.009231429547071457,
      "rewards/margins": -0.0193635243922472,
      "rewards/rejected": 0.02859494648873806,
      "step": 495
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.608050695227385e-05,
      "logits/chosen": -2.0139918327331543,
      "logits/rejected": -2.0279946327209473,
      "logps/chosen": -163.32350158691406,
      "logps/rejected": -148.53485107421875,
      "loss": 0.7111,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04781512916088104,
      "rewards/margins": 0.12081319838762283,
      "rewards/rejected": -0.16862833499908447,
      "step": 496
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.606122412799857e-05,
      "logits/chosen": -2.3113937377929688,
      "logits/rejected": -2.31269907951355,
      "logps/chosen": -147.7089080810547,
      "logps/rejected": -147.32814025878906,
      "loss": 0.6941,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2833206355571747,
      "rewards/margins": 0.048162560909986496,
      "rewards/rejected": -0.3314831852912903,
      "step": 497
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.6041898042162764e-05,
      "logits/chosen": -2.3534443378448486,
      "logits/rejected": -2.3750576972961426,
      "logps/chosen": -179.6420440673828,
      "logps/rejected": -186.58526611328125,
      "loss": 0.8517,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.15785303711891174,
      "rewards/margins": -0.20781296491622925,
      "rewards/rejected": 0.049959927797317505,
      "step": 498
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.602252873446386e-05,
      "logits/chosen": -1.7936934232711792,
      "logits/rejected": -1.7282207012176514,
      "logps/chosen": -171.0880584716797,
      "logps/rejected": -180.4699249267578,
      "loss": 0.7747,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.37305474281311035,
      "rewards/margins": 0.08523893356323242,
      "rewards/rejected": -0.45829373598098755,
      "step": 499
    },
    {
      "epoch": 0.65,
      "learning_rate": 4.60031162446881e-05,
      "logits/chosen": -2.154116153717041,
      "logits/rejected": -2.1634771823883057,
      "logps/chosen": -142.67947387695312,
      "logps/rejected": -141.00306701660156,
      "loss": 0.651,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.04277772828936577,
      "rewards/margins": 0.16890141367912292,
      "rewards/rejected": -0.12612366676330566,
      "step": 500
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.5983660612710365e-05,
      "logits/chosen": -2.0763444900512695,
      "logits/rejected": -2.0053396224975586,
      "logps/chosen": -151.91763305664062,
      "logps/rejected": -144.047607421875,
      "loss": 0.8691,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.48068270087242126,
      "rewards/margins": -0.29285871982574463,
      "rewards/rejected": -0.18782396614551544,
      "step": 501
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.596416187849423e-05,
      "logits/chosen": -2.0705764293670654,
      "logits/rejected": -2.0982978343963623,
      "logps/chosen": -132.6533203125,
      "logps/rejected": -136.65093994140625,
      "loss": 0.5925,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.2096860408782959,
      "rewards/margins": 0.35339686274528503,
      "rewards/rejected": -0.14371080696582794,
      "step": 502
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.5944620082091745e-05,
      "logits/chosen": -1.9696085453033447,
      "logits/rejected": -2.015946865081787,
      "logps/chosen": -253.59783935546875,
      "logps/rejected": -241.3708038330078,
      "loss": 0.7486,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.22850322723388672,
      "rewards/margins": -0.06344452500343323,
      "rewards/rejected": -0.16505871713161469,
      "step": 503
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.5925035263643444e-05,
      "logits/chosen": -2.2930264472961426,
      "logits/rejected": -2.2867448329925537,
      "logps/chosen": -167.2974090576172,
      "logps/rejected": -155.6397247314453,
      "loss": 0.7972,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.23387077450752258,
      "rewards/margins": -0.13073894381523132,
      "rewards/rejected": -0.10313184559345245,
      "step": 504
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.5905407463378225e-05,
      "logits/chosen": -2.336209774017334,
      "logits/rejected": -2.3158388137817383,
      "logps/chosen": -153.33377075195312,
      "logps/rejected": -149.57073974609375,
      "loss": 0.6428,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.29021593928337097,
      "rewards/margins": 0.20943422615528107,
      "rewards/rejected": 0.08078169822692871,
      "step": 505
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.588573672161326e-05,
      "logits/chosen": -1.9931033849716187,
      "logits/rejected": -1.9577481746673584,
      "logps/chosen": -165.14727783203125,
      "logps/rejected": -179.18228149414062,
      "loss": 0.7458,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.25228357315063477,
      "rewards/margins": -0.07357550412416458,
      "rewards/rejected": -0.1787080615758896,
      "step": 506
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.586602307875396e-05,
      "logits/chosen": -2.2065775394439697,
      "logits/rejected": -2.18172550201416,
      "logps/chosen": -171.14437866210938,
      "logps/rejected": -185.22750854492188,
      "loss": 0.799,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.23644599318504333,
      "rewards/margins": -0.07532497495412827,
      "rewards/rejected": -0.16112102568149567,
      "step": 507
    },
    {
      "epoch": 0.66,
      "learning_rate": 4.5846266575293816e-05,
      "logits/chosen": -2.1658072471618652,
      "logits/rejected": -2.1649296283721924,
      "logps/chosen": -180.37255859375,
      "logps/rejected": -158.17051696777344,
      "loss": 0.7103,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.09764609485864639,
      "rewards/margins": 0.07636680454015732,
      "rewards/rejected": -0.1740128993988037,
      "step": 508
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.582646725181441e-05,
      "logits/chosen": -1.7266638278961182,
      "logits/rejected": -1.8396620750427246,
      "logps/chosen": -129.93885803222656,
      "logps/rejected": -139.12106323242188,
      "loss": 0.6701,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.10675990581512451,
      "rewards/margins": 0.09391005337238312,
      "rewards/rejected": -0.20066994428634644,
      "step": 509
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.580662514898522e-05,
      "logits/chosen": -2.210435628890991,
      "logits/rejected": -2.176440477371216,
      "logps/chosen": -161.60562133789062,
      "logps/rejected": -143.5146484375,
      "loss": 0.6319,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.23548352718353271,
      "rewards/margins": 0.1876177191734314,
      "rewards/rejected": -0.4231012463569641,
      "step": 510
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.5786740307563636e-05,
      "logits/chosen": -2.292863130569458,
      "logits/rejected": -2.286449432373047,
      "logps/chosen": -149.38560485839844,
      "logps/rejected": -152.18753051757812,
      "loss": 0.694,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1346094161272049,
      "rewards/margins": 0.04607778787612915,
      "rewards/rejected": -0.18068721890449524,
      "step": 511
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.576681276839483e-05,
      "logits/chosen": -2.000340461730957,
      "logits/rejected": -1.9955222606658936,
      "logps/chosen": -169.0735321044922,
      "logps/rejected": -171.15048217773438,
      "loss": 0.7291,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1468675434589386,
      "rewards/margins": 0.021292299032211304,
      "rewards/rejected": -0.1681598424911499,
      "step": 512
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.574684257241168e-05,
      "logits/chosen": -1.7635064125061035,
      "logits/rejected": -1.7762371301651,
      "logps/chosen": -179.9252471923828,
      "logps/rejected": -179.78445434570312,
      "loss": 0.7814,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.15447980165481567,
      "rewards/margins": -0.054616428911685944,
      "rewards/rejected": -0.09986338019371033,
      "step": 513
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.572682976063468e-05,
      "logits/chosen": -2.17221736907959,
      "logits/rejected": -2.205554485321045,
      "logps/chosen": -124.07170104980469,
      "logps/rejected": -144.3170928955078,
      "loss": 0.835,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.1368752419948578,
      "rewards/margins": -0.1436615288257599,
      "rewards/rejected": 0.2805367410182953,
      "step": 514
    },
    {
      "epoch": 0.67,
      "learning_rate": 4.5706774374171854e-05,
      "logits/chosen": -2.003601551055908,
      "logits/rejected": -2.0354526042938232,
      "logps/chosen": -201.1471710205078,
      "logps/rejected": -220.0992431640625,
      "loss": 0.8975,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.33305513858795166,
      "rewards/margins": -0.3173186182975769,
      "rewards/rejected": -0.01573648676276207,
      "step": 515
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.56866764542187e-05,
      "logits/chosen": -2.144970417022705,
      "logits/rejected": -2.187018394470215,
      "logps/chosen": -156.69705200195312,
      "logps/rejected": -157.2908477783203,
      "loss": 0.6765,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.24758291244506836,
      "rewards/margins": 0.09244808554649353,
      "rewards/rejected": 0.15513482689857483,
      "step": 516
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.566653604205805e-05,
      "logits/chosen": -2.15089750289917,
      "logits/rejected": -2.1836040019989014,
      "logps/chosen": -166.6836395263672,
      "logps/rejected": -181.80368041992188,
      "loss": 0.7843,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2800065279006958,
      "rewards/margins": -0.08594866096973419,
      "rewards/rejected": 0.3659551739692688,
      "step": 517
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.5646353179060057e-05,
      "logits/chosen": -2.251128673553467,
      "logits/rejected": -2.24637770652771,
      "logps/chosen": -137.0120849609375,
      "logps/rejected": -135.96041870117188,
      "loss": 0.6631,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.20585085451602936,
      "rewards/margins": 0.15107136964797974,
      "rewards/rejected": 0.054779477417469025,
      "step": 518
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.562612790668204e-05,
      "logits/chosen": -1.813207983970642,
      "logits/rejected": -1.8710843324661255,
      "logps/chosen": -126.45897674560547,
      "logps/rejected": -127.25151062011719,
      "loss": 0.6969,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.2428382784128189,
      "rewards/margins": 0.050096381455659866,
      "rewards/rejected": 0.19274193048477173,
      "step": 519
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.560586026646845e-05,
      "logits/chosen": -2.1233975887298584,
      "logits/rejected": -2.1502273082733154,
      "logps/chosen": -141.46368408203125,
      "logps/rejected": -143.70501708984375,
      "loss": 0.6942,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05234923213720322,
      "rewards/margins": 0.039612069725990295,
      "rewards/rejected": 0.012737158685922623,
      "step": 520
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.558555030005075e-05,
      "logits/chosen": -1.9741415977478027,
      "logits/rejected": -1.9955835342407227,
      "logps/chosen": -144.7069549560547,
      "logps/rejected": -167.02902221679688,
      "loss": 0.7312,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05080757290124893,
      "rewards/margins": -0.03744647279381752,
      "rewards/rejected": 0.08825404942035675,
      "step": 521
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.556519804914736e-05,
      "logits/chosen": -2.0981431007385254,
      "logits/rejected": -2.073946714401245,
      "logps/chosen": -225.3271942138672,
      "logps/rejected": -232.33668518066406,
      "loss": 0.7173,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.12561050057411194,
      "rewards/margins": 0.03423985838890076,
      "rewards/rejected": -0.15985038876533508,
      "step": 522
    },
    {
      "epoch": 0.68,
      "learning_rate": 4.554480355556354e-05,
      "logits/chosen": -1.8198878765106201,
      "logits/rejected": -1.7702445983886719,
      "logps/chosen": -148.35325622558594,
      "logps/rejected": -151.1799774169922,
      "loss": 0.5943,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.26486697793006897,
      "rewards/margins": 0.25646913051605225,
      "rewards/rejected": 0.00839783065021038,
      "step": 523
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.552436686119134e-05,
      "logits/chosen": -1.8180818557739258,
      "logits/rejected": -1.764922857284546,
      "logps/chosen": -175.82241821289062,
      "logps/rejected": -188.15457153320312,
      "loss": 0.6774,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.026587432250380516,
      "rewards/margins": 0.06728048622608185,
      "rewards/rejected": -0.0938679426908493,
      "step": 524
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.550388800800948e-05,
      "logits/chosen": -1.7546145915985107,
      "logits/rejected": -1.7742228507995605,
      "logps/chosen": -152.79208374023438,
      "logps/rejected": -140.30728149414062,
      "loss": 0.7556,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.02463456243276596,
      "rewards/margins": -0.02168111503124237,
      "rewards/rejected": 0.046315666288137436,
      "step": 525
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.548336703808328e-05,
      "logits/chosen": -1.8469312191009521,
      "logits/rejected": -1.8778002262115479,
      "logps/chosen": -126.43699645996094,
      "logps/rejected": -146.99752807617188,
      "loss": 0.7736,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.17384478449821472,
      "rewards/margins": -0.11481676995754242,
      "rewards/rejected": -0.059028007090091705,
      "step": 526
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.546280399356457e-05,
      "logits/chosen": -1.8530418872833252,
      "logits/rejected": -1.927619218826294,
      "logps/chosen": -175.30499267578125,
      "logps/rejected": -165.12033081054688,
      "loss": 0.8325,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.1547343134880066,
      "rewards/margins": -0.2065506875514984,
      "rewards/rejected": 0.05181637406349182,
      "step": 527
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.54421989166916e-05,
      "logits/chosen": -2.190769910812378,
      "logits/rejected": -2.173021078109741,
      "logps/chosen": -148.19883728027344,
      "logps/rejected": -149.60968017578125,
      "loss": 0.6945,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.1685972660779953,
      "rewards/margins": 0.083544060587883,
      "rewards/rejected": 0.0850532054901123,
      "step": 528
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.542155184978898e-05,
      "logits/chosen": -2.227403402328491,
      "logits/rejected": -2.2359485626220703,
      "logps/chosen": -129.83944702148438,
      "logps/rejected": -130.6429443359375,
      "loss": 0.6671,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.20258253812789917,
      "rewards/margins": 0.09699730575084686,
      "rewards/rejected": 0.1055852472782135,
      "step": 529
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.540086283526754e-05,
      "logits/chosen": -2.176431179046631,
      "logits/rejected": -2.1221935749053955,
      "logps/chosen": -148.75912475585938,
      "logps/rejected": -121.70980834960938,
      "loss": 0.6792,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11940637975931168,
      "rewards/margins": 0.08864616602659225,
      "rewards/rejected": 0.030760202556848526,
      "step": 530
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.538013191562431e-05,
      "logits/chosen": -2.2914586067199707,
      "logits/rejected": -2.2601258754730225,
      "logps/chosen": -185.4001922607422,
      "logps/rejected": -187.08941650390625,
      "loss": 0.6852,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.008667275309562683,
      "rewards/margins": 0.06690745055675507,
      "rewards/rejected": -0.05824017524719238,
      "step": 531
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.5359359133442356e-05,
      "logits/chosen": -2.1340489387512207,
      "logits/rejected": -2.1484596729278564,
      "logps/chosen": -145.6934814453125,
      "logps/rejected": -157.48910522460938,
      "loss": 0.6541,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.03526604175567627,
      "rewards/margins": 0.22745643556118011,
      "rewards/rejected": -0.19219039380550385,
      "step": 532
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.533854453139077e-05,
      "logits/chosen": -2.163139581680298,
      "logits/rejected": -2.1732969284057617,
      "logps/chosen": -157.28152465820312,
      "logps/rejected": -164.52548217773438,
      "loss": 0.5849,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14357003569602966,
      "rewards/margins": 0.33174028992652893,
      "rewards/rejected": -0.18817025423049927,
      "step": 533
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.5317688152224515e-05,
      "logits/chosen": -2.232987642288208,
      "logits/rejected": -2.2299559116363525,
      "logps/chosen": -200.51223754882812,
      "logps/rejected": -193.10623168945312,
      "loss": 0.7663,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08180832862854004,
      "rewards/margins": -0.07060222327709198,
      "rewards/rejected": 0.1524105668067932,
      "step": 534
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.52967900387844e-05,
      "logits/chosen": -2.139064073562622,
      "logits/rejected": -2.153592586517334,
      "logps/chosen": -150.04501342773438,
      "logps/rejected": -153.50726318359375,
      "loss": 0.8413,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.004394152667373419,
      "rewards/margins": -0.2386518269777298,
      "rewards/rejected": 0.24304598569869995,
      "step": 535
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.5275850233996925e-05,
      "logits/chosen": -2.081514835357666,
      "logits/rejected": -2.029918670654297,
      "logps/chosen": -152.26541137695312,
      "logps/rejected": -168.11135864257812,
      "loss": 0.6554,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.15842419862747192,
      "rewards/margins": 0.11571840941905975,
      "rewards/rejected": 0.042705781757831573,
      "step": 536
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.525486878087426e-05,
      "logits/chosen": -2.0249204635620117,
      "logits/rejected": -1.9895412921905518,
      "logps/chosen": -148.14431762695312,
      "logps/rejected": -138.2922821044922,
      "loss": 0.7594,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.12210666388273239,
      "rewards/margins": -0.07970865815877914,
      "rewards/rejected": 0.20181530714035034,
      "step": 537
    },
    {
      "epoch": 0.7,
      "learning_rate": 4.523384572251409e-05,
      "logits/chosen": -2.079011917114258,
      "logits/rejected": -2.151048421859741,
      "logps/chosen": -172.8366241455078,
      "logps/rejected": -196.3922576904297,
      "loss": 0.7753,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.05615053325891495,
      "rewards/margins": -0.0960412472486496,
      "rewards/rejected": 0.15219178795814514,
      "step": 538
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.52127811020996e-05,
      "logits/chosen": -2.2182424068450928,
      "logits/rejected": -2.2047057151794434,
      "logps/chosen": -143.86557006835938,
      "logps/rejected": -155.3209686279297,
      "loss": 0.6784,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.018200304359197617,
      "rewards/margins": 0.08800992369651794,
      "rewards/rejected": -0.10621023923158646,
      "step": 539
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.5191674962899314e-05,
      "logits/chosen": -1.9326967000961304,
      "logits/rejected": -1.9407259225845337,
      "logps/chosen": -166.248291015625,
      "logps/rejected": -184.2652130126953,
      "loss": 0.7111,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.07094278186559677,
      "rewards/margins": 0.1123301088809967,
      "rewards/rejected": -0.18327289819717407,
      "step": 540
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.5170527348267054e-05,
      "logits/chosen": -2.1330301761627197,
      "logits/rejected": -2.218416690826416,
      "logps/chosen": -138.27850341796875,
      "logps/rejected": -139.3997344970703,
      "loss": 0.6661,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.176968052983284,
      "rewards/margins": 0.11333853006362915,
      "rewards/rejected": -0.29030656814575195,
      "step": 541
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.5149338301641845e-05,
      "logits/chosen": -2.203751564025879,
      "logits/rejected": -2.177703380584717,
      "logps/chosen": -153.626708984375,
      "logps/rejected": -145.74542236328125,
      "loss": 0.8742,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.1450812965631485,
      "rewards/margins": -0.26044121384620667,
      "rewards/rejected": 0.11535991728305817,
      "step": 542
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.512810786654779e-05,
      "logits/chosen": -2.061079502105713,
      "logits/rejected": -2.138723611831665,
      "logps/chosen": -163.50033569335938,
      "logps/rejected": -169.90225219726562,
      "loss": 0.7113,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.12472327053546906,
      "rewards/margins": 0.01421796903014183,
      "rewards/rejected": -0.13894124329090118,
      "step": 543
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.510683608659403e-05,
      "logits/chosen": -1.9471606016159058,
      "logits/rejected": -1.951507806777954,
      "logps/chosen": -153.19105529785156,
      "logps/rejected": -183.1653289794922,
      "loss": 0.7854,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2256731539964676,
      "rewards/margins": -0.1194252148270607,
      "rewards/rejected": -0.1062479168176651,
      "step": 544
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.508552300547463e-05,
      "logits/chosen": -1.9512101411819458,
      "logits/rejected": -2.0366945266723633,
      "logps/chosen": -162.8254852294922,
      "logps/rejected": -170.9732666015625,
      "loss": 0.6963,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13340815901756287,
      "rewards/margins": 0.08452873677015305,
      "rewards/rejected": -0.2179369032382965,
      "step": 545
    },
    {
      "epoch": 0.71,
      "learning_rate": 4.506416866696848e-05,
      "logits/chosen": -2.29795503616333,
      "logits/rejected": -2.2391650676727295,
      "logps/chosen": -157.7142791748047,
      "logps/rejected": -160.1687469482422,
      "loss": 0.7126,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.1038050726056099,
      "rewards/margins": 0.07325511425733566,
      "rewards/rejected": 0.030549969524145126,
      "step": 546
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.504277311493922e-05,
      "logits/chosen": -2.0046989917755127,
      "logits/rejected": -2.0597028732299805,
      "logps/chosen": -161.10891723632812,
      "logps/rejected": -154.01663208007812,
      "loss": 0.6706,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.03068912774324417,
      "rewards/margins": 0.09607706218957901,
      "rewards/rejected": -0.126766175031662,
      "step": 547
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.502133639333516e-05,
      "logits/chosen": -2.2434914112091064,
      "logits/rejected": -2.2011220455169678,
      "logps/chosen": -166.30824279785156,
      "logps/rejected": -166.396240234375,
      "loss": 0.7998,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.23776453733444214,
      "rewards/margins": -0.1319459229707718,
      "rewards/rejected": -0.10581861436367035,
      "step": 548
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.499985854618915e-05,
      "logits/chosen": -2.00533127784729,
      "logits/rejected": -1.9807076454162598,
      "logps/chosen": -171.72430419921875,
      "logps/rejected": -176.17196655273438,
      "loss": 0.7283,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.1258818507194519,
      "rewards/margins": -0.023614590987563133,
      "rewards/rejected": -0.10226726531982422,
      "step": 549
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.497833961761855e-05,
      "logits/chosen": -1.9163005352020264,
      "logits/rejected": -1.9533987045288086,
      "logps/chosen": -187.05703735351562,
      "logps/rejected": -160.68228149414062,
      "loss": 0.7811,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.18283602595329285,
      "rewards/margins": -0.12604478001594543,
      "rewards/rejected": -0.05679125338792801,
      "step": 550
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.495677965182506e-05,
      "logits/chosen": -1.7719175815582275,
      "logits/rejected": -1.7788423299789429,
      "logps/chosen": -149.80589294433594,
      "logps/rejected": -148.5959014892578,
      "loss": 0.7087,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.0498424731194973,
      "rewards/margins": 0.04123033583164215,
      "rewards/rejected": 0.00861213356256485,
      "step": 551
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.4935178693094714e-05,
      "logits/chosen": -1.8553767204284668,
      "logits/rejected": -1.8368808031082153,
      "logps/chosen": -179.26145935058594,
      "logps/rejected": -164.9579620361328,
      "loss": 0.8264,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5210792422294617,
      "rewards/margins": -0.1817505955696106,
      "rewards/rejected": -0.3393285870552063,
      "step": 552
    },
    {
      "epoch": 0.72,
      "learning_rate": 4.491353678579774e-05,
      "logits/chosen": -2.287044048309326,
      "logits/rejected": -2.296315908432007,
      "logps/chosen": -154.74530029296875,
      "logps/rejected": -161.77780151367188,
      "loss": 0.7811,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.222887322306633,
      "rewards/margins": -0.1348755806684494,
      "rewards/rejected": -0.0880117416381836,
      "step": 553
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.489185397438845e-05,
      "logits/chosen": -2.1219000816345215,
      "logits/rejected": -2.1018307209014893,
      "logps/chosen": -152.45147705078125,
      "logps/rejected": -163.04336547851562,
      "loss": 0.7666,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.2625100910663605,
      "rewards/margins": -0.10655740648508072,
      "rewards/rejected": -0.15595270693302155,
      "step": 554
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.4870130303405214e-05,
      "logits/chosen": -2.108427047729492,
      "logits/rejected": -2.1353774070739746,
      "logps/chosen": -117.01795196533203,
      "logps/rejected": -126.68714141845703,
      "loss": 0.6582,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.13066193461418152,
      "rewards/margins": 0.10399821400642395,
      "rewards/rejected": 0.026663724333047867,
      "step": 555
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.484836581747032e-05,
      "logits/chosen": -2.193756580352783,
      "logits/rejected": -2.219733953475952,
      "logps/chosen": -152.94854736328125,
      "logps/rejected": -157.80963134765625,
      "loss": 0.6522,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12146392464637756,
      "rewards/margins": 0.1416078507900238,
      "rewards/rejected": -0.26307177543640137,
      "step": 556
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.4826560561289865e-05,
      "logits/chosen": -2.087712526321411,
      "logits/rejected": -2.0898258686065674,
      "logps/chosen": -172.42575073242188,
      "logps/rejected": -163.4795379638672,
      "loss": 0.628,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.22759903967380524,
      "rewards/margins": 0.20539291203022003,
      "rewards/rejected": -0.4329919219017029,
      "step": 557
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.4804714579653736e-05,
      "logits/chosen": -2.2692031860351562,
      "logits/rejected": -2.326690435409546,
      "logps/chosen": -161.824951171875,
      "logps/rejected": -154.09632873535156,
      "loss": 0.7904,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4080825448036194,
      "rewards/margins": -0.13219794631004333,
      "rewards/rejected": -0.27588459849357605,
      "step": 558
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.4782827917435454e-05,
      "logits/chosen": -1.4946506023406982,
      "logits/rejected": -1.4943220615386963,
      "logps/chosen": -169.450927734375,
      "logps/rejected": -177.3900146484375,
      "loss": 0.715,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.09592101722955704,
      "rewards/margins": -0.032054100185632706,
      "rewards/rejected": -0.06386692076921463,
      "step": 559
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.4760900619592085e-05,
      "logits/chosen": -2.043163537979126,
      "logits/rejected": -2.1061294078826904,
      "logps/chosen": -146.00006103515625,
      "logps/rejected": -167.94851684570312,
      "loss": 0.7728,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.056870993226766586,
      "rewards/margins": -0.011001009494066238,
      "rewards/rejected": -0.045869968831539154,
      "step": 560
    },
    {
      "epoch": 0.73,
      "learning_rate": 4.4738932731164194e-05,
      "logits/chosen": -2.10860538482666,
      "logits/rejected": -2.0873773097991943,
      "logps/chosen": -147.17306518554688,
      "logps/rejected": -149.69338989257812,
      "loss": 0.673,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06803455948829651,
      "rewards/margins": 0.09854596853256226,
      "rewards/rejected": -0.16658052802085876,
      "step": 561
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.47169242972757e-05,
      "logits/chosen": -2.1042397022247314,
      "logits/rejected": -2.107679605484009,
      "logps/chosen": -206.35293579101562,
      "logps/rejected": -179.4486541748047,
      "loss": 0.7299,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.17581839859485626,
      "rewards/margins": -0.013735771179199219,
      "rewards/rejected": -0.16208262741565704,
      "step": 562
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.469487536313381e-05,
      "logits/chosen": -1.9570105075836182,
      "logits/rejected": -1.9454288482666016,
      "logps/chosen": -165.0225067138672,
      "logps/rejected": -182.94992065429688,
      "loss": 0.6866,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18598608672618866,
      "rewards/margins": 0.06062725931406021,
      "rewards/rejected": -0.24661336839199066,
      "step": 563
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.467278597402894e-05,
      "logits/chosen": -2.0907437801361084,
      "logits/rejected": -2.0541324615478516,
      "logps/chosen": -186.67782592773438,
      "logps/rejected": -194.32305908203125,
      "loss": 0.7212,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.26842382550239563,
      "rewards/margins": 0.008616073057055473,
      "rewards/rejected": -0.27703991532325745,
      "step": 564
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.465065617533457e-05,
      "logits/chosen": -2.0557780265808105,
      "logits/rejected": -2.066563129425049,
      "logps/chosen": -146.12057495117188,
      "logps/rejected": -144.6167449951172,
      "loss": 0.9087,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.5873019695281982,
      "rewards/margins": -0.3313911557197571,
      "rewards/rejected": -0.25591081380844116,
      "step": 565
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.462848601250722e-05,
      "logits/chosen": -2.1823415756225586,
      "logits/rejected": -2.1932756900787354,
      "logps/chosen": -151.8548126220703,
      "logps/rejected": -147.4888916015625,
      "loss": 0.6847,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.24860316514968872,
      "rewards/margins": 0.11341544985771179,
      "rewards/rejected": -0.3620185852050781,
      "step": 566
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.4606275531086295e-05,
      "logits/chosen": -1.800735354423523,
      "logits/rejected": -1.7712013721466064,
      "logps/chosen": -145.9650115966797,
      "logps/rejected": -163.93515014648438,
      "loss": 0.7801,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.49170562624931335,
      "rewards/margins": -0.12783843278884888,
      "rewards/rejected": -0.3638671636581421,
      "step": 567
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.4584024776694035e-05,
      "logits/chosen": -1.8931790590286255,
      "logits/rejected": -1.8744131326675415,
      "logps/chosen": -208.36671447753906,
      "logps/rejected": -222.00416564941406,
      "loss": 0.6506,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.040707264095544815,
      "rewards/margins": 0.14632365107536316,
      "rewards/rejected": -0.18703092634677887,
      "step": 568
    },
    {
      "epoch": 0.74,
      "learning_rate": 4.45617337950354e-05,
      "logits/chosen": -1.761922836303711,
      "logits/rejected": -1.7843868732452393,
      "logps/chosen": -153.74264526367188,
      "logps/rejected": -178.4879150390625,
      "loss": 0.7251,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.10153071582317352,
      "rewards/margins": 0.11845846474170685,
      "rewards/rejected": -0.21998921036720276,
      "step": 569
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.453940263189797e-05,
      "logits/chosen": -2.2519378662109375,
      "logits/rejected": -2.1770126819610596,
      "logps/chosen": -142.41217041015625,
      "logps/rejected": -133.3799285888672,
      "loss": 0.709,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.19167102873325348,
      "rewards/margins": 0.01131703332066536,
      "rewards/rejected": -0.20298807322978973,
      "step": 570
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.4517031333151874e-05,
      "logits/chosen": -2.078045606613159,
      "logits/rejected": -2.046724557876587,
      "logps/chosen": -137.99807739257812,
      "logps/rejected": -137.1576385498047,
      "loss": 0.8089,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5403813123703003,
      "rewards/margins": -0.16598506271839142,
      "rewards/rejected": -0.37439629435539246,
      "step": 571
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.449461994474968e-05,
      "logits/chosen": -1.9291788339614868,
      "logits/rejected": -1.961287260055542,
      "logps/chosen": -190.75672912597656,
      "logps/rejected": -210.29931640625,
      "loss": 0.7873,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.06664823740720749,
      "rewards/margins": -0.008556842803955078,
      "rewards/rejected": -0.05809139460325241,
      "step": 572
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.44721685127263e-05,
      "logits/chosen": -2.2334282398223877,
      "logits/rejected": -2.1782572269439697,
      "logps/chosen": -164.24710083007812,
      "logps/rejected": -161.57275390625,
      "loss": 0.8631,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.24856142699718475,
      "rewards/margins": -0.2524217367172241,
      "rewards/rejected": 0.003860296681523323,
      "step": 573
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.4449677083198896e-05,
      "logits/chosen": -2.198751211166382,
      "logits/rejected": -2.1961047649383545,
      "logps/chosen": -139.06393432617188,
      "logps/rejected": -136.85275268554688,
      "loss": 0.6852,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.24132420122623444,
      "rewards/margins": 0.04328273981809616,
      "rewards/rejected": -0.2846069633960724,
      "step": 574
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.4427145702366804e-05,
      "logits/chosen": -2.167025327682495,
      "logits/rejected": -2.1032488346099854,
      "logps/chosen": -150.00331115722656,
      "logps/rejected": -143.5449981689453,
      "loss": 0.805,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3796214461326599,
      "rewards/margins": -0.16332601010799408,
      "rewards/rejected": -0.21629548072814941,
      "step": 575
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.440457441651139e-05,
      "logits/chosen": -2.0747876167297363,
      "logits/rejected": -2.0561654567718506,
      "logps/chosen": -189.77783203125,
      "logps/rejected": -190.18592834472656,
      "loss": 0.6855,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1576140820980072,
      "rewards/margins": 0.06496872752904892,
      "rewards/rejected": -0.2225828319787979,
      "step": 576
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.4381963271996044e-05,
      "logits/chosen": -1.8249603509902954,
      "logits/rejected": -1.7742782831192017,
      "logps/chosen": -165.38795471191406,
      "logps/rejected": -166.17652893066406,
      "loss": 0.757,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.25819915533065796,
      "rewards/margins": 0.011954933404922485,
      "rewards/rejected": -0.27015408873558044,
      "step": 577
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.435931231526597e-05,
      "logits/chosen": -2.0663070678710938,
      "logits/rejected": -2.095902919769287,
      "logps/chosen": -168.3959197998047,
      "logps/rejected": -168.69862365722656,
      "loss": 0.8164,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5112836360931396,
      "rewards/margins": -0.09502200782299042,
      "rewards/rejected": -0.4162616431713104,
      "step": 578
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.433662159284818e-05,
      "logits/chosen": -1.9458318948745728,
      "logits/rejected": -1.9755809307098389,
      "logps/chosen": -158.89151000976562,
      "logps/rejected": -181.25265502929688,
      "loss": 0.6813,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.40285855531692505,
      "rewards/margins": 0.05986136570572853,
      "rewards/rejected": -0.4627199172973633,
      "step": 579
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.4313891151351375e-05,
      "logits/chosen": -1.8997300863265991,
      "logits/rejected": -1.9370012283325195,
      "logps/chosen": -189.06930541992188,
      "logps/rejected": -198.09793090820312,
      "loss": 0.725,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23561373353004456,
      "rewards/margins": 0.04797305539250374,
      "rewards/rejected": -0.2835868000984192,
      "step": 580
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.429112103746582e-05,
      "logits/chosen": -2.2178287506103516,
      "logits/rejected": -2.218778133392334,
      "logps/chosen": -225.8673095703125,
      "logps/rejected": -211.37353515625,
      "loss": 0.6963,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.43705853819847107,
      "rewards/margins": 0.05578887462615967,
      "rewards/rejected": -0.49284741282463074,
      "step": 581
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.4268311297963295e-05,
      "logits/chosen": -1.8671170473098755,
      "logits/rejected": -1.928563117980957,
      "logps/chosen": -168.80242919921875,
      "logps/rejected": -170.85487365722656,
      "loss": 0.9331,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6543461084365845,
      "rewards/margins": -0.21871261298656464,
      "rewards/rejected": -0.43563348054885864,
      "step": 582
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.4245461979696937e-05,
      "logits/chosen": -2.1317386627197266,
      "logits/rejected": -2.069695234298706,
      "logps/chosen": -170.15565490722656,
      "logps/rejected": -172.81195068359375,
      "loss": 0.7135,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.37158751487731934,
      "rewards/margins": 0.09176269173622131,
      "rewards/rejected": -0.46335020661354065,
      "step": 583
    },
    {
      "epoch": 0.76,
      "learning_rate": 4.422257312960123e-05,
      "logits/chosen": -2.1730875968933105,
      "logits/rejected": -2.2167327404022217,
      "logps/chosen": -165.11517333984375,
      "logps/rejected": -160.7394256591797,
      "loss": 0.666,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.11963924765586853,
      "rewards/margins": 0.12844307720661163,
      "rewards/rejected": -0.24808233976364136,
      "step": 584
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.419964479469182e-05,
      "logits/chosen": -2.059325933456421,
      "logits/rejected": -2.017867088317871,
      "logps/chosen": -155.38075256347656,
      "logps/rejected": -140.58778381347656,
      "loss": 0.5993,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.31128260493278503,
      "rewards/margins": 0.24781286716461182,
      "rewards/rejected": -0.5590954422950745,
      "step": 585
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.417667702206548e-05,
      "logits/chosen": -1.8881521224975586,
      "logits/rejected": -1.952433705329895,
      "logps/chosen": -161.21728515625,
      "logps/rejected": -170.68008422851562,
      "loss": 0.7572,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.32663464546203613,
      "rewards/margins": -0.015847772359848022,
      "rewards/rejected": -0.3107869029045105,
      "step": 586
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.415366985889998e-05,
      "logits/chosen": -2.0811541080474854,
      "logits/rejected": -2.0584499835968018,
      "logps/chosen": -153.13113403320312,
      "logps/rejected": -144.1329803466797,
      "loss": 0.7128,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.24523815512657166,
      "rewards/margins": 0.0006711352616548538,
      "rewards/rejected": -0.24590928852558136,
      "step": 587
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.413062335245402e-05,
      "logits/chosen": -2.009754180908203,
      "logits/rejected": -2.03741717338562,
      "logps/chosen": -178.30477905273438,
      "logps/rejected": -169.07321166992188,
      "loss": 0.6164,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.011600993573665619,
      "rewards/margins": 0.23952721059322357,
      "rewards/rejected": -0.251128226518631,
      "step": 588
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.410753755006708e-05,
      "logits/chosen": -2.0812978744506836,
      "logits/rejected": -2.0661728382110596,
      "logps/chosen": -205.08026123046875,
      "logps/rejected": -189.98153686523438,
      "loss": 0.6828,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.22344040870666504,
      "rewards/margins": 0.08454579859972,
      "rewards/rejected": -0.30798622965812683,
      "step": 589
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.408441249915938e-05,
      "logits/chosen": -2.2237741947174072,
      "logits/rejected": -2.2071006298065186,
      "logps/chosen": -145.8521728515625,
      "logps/rejected": -153.61093139648438,
      "loss": 0.6731,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.031120114028453827,
      "rewards/margins": 0.07498609274625778,
      "rewards/rejected": -0.10610620677471161,
      "step": 590
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.4061248247231776e-05,
      "logits/chosen": -2.0926318168640137,
      "logits/rejected": -2.064544916152954,
      "logps/chosen": -200.65924072265625,
      "logps/rejected": -201.64376831054688,
      "loss": 0.7178,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2164732813835144,
      "rewards/margins": 0.056919313967227936,
      "rewards/rejected": -0.27339258790016174,
      "step": 591
    },
    {
      "epoch": 0.77,
      "learning_rate": 4.4038044841865614e-05,
      "logits/chosen": -1.9554996490478516,
      "logits/rejected": -1.9424071311950684,
      "logps/chosen": -165.02134704589844,
      "logps/rejected": -169.86578369140625,
      "loss": 0.6787,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.12406764924526215,
      "rewards/margins": 0.09511314332485199,
      "rewards/rejected": -0.21918082237243652,
      "step": 592
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.401480233072268e-05,
      "logits/chosen": -2.166558265686035,
      "logits/rejected": -2.2012734413146973,
      "logps/chosen": -203.3728485107422,
      "logps/rejected": -207.2845001220703,
      "loss": 0.7391,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.14014330506324768,
      "rewards/margins": -0.0016260165721178055,
      "rewards/rejected": -0.13851726055145264,
      "step": 593
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.399152076154509e-05,
      "logits/chosen": -1.908893346786499,
      "logits/rejected": -1.9617127180099487,
      "logps/chosen": -154.35598754882812,
      "logps/rejected": -156.9701690673828,
      "loss": 0.775,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.09893524646759033,
      "rewards/margins": -0.09441002458333969,
      "rewards/rejected": -0.004525229334831238,
      "step": 594
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.396820018215518e-05,
      "logits/chosen": -1.9621710777282715,
      "logits/rejected": -2.016408920288086,
      "logps/chosen": -178.7270965576172,
      "logps/rejected": -200.99447631835938,
      "loss": 0.6945,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3592956066131592,
      "rewards/margins": 0.08869167417287827,
      "rewards/rejected": -0.44798728823661804,
      "step": 595
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.394484064045542e-05,
      "logits/chosen": -2.2477211952209473,
      "logits/rejected": -2.2041501998901367,
      "logps/chosen": -233.4781494140625,
      "logps/rejected": -218.37103271484375,
      "loss": 0.8302,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.027775531634688377,
      "rewards/margins": -0.17972132563591003,
      "rewards/rejected": 0.1519457995891571,
      "step": 596
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.392144218442831e-05,
      "logits/chosen": -1.885699987411499,
      "logits/rejected": -1.8928428888320923,
      "logps/chosen": -147.02767944335938,
      "logps/rejected": -141.90377807617188,
      "loss": 0.7493,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.34831514954566956,
      "rewards/margins": -0.05675073340535164,
      "rewards/rejected": -0.291564404964447,
      "step": 597
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.3898004862136286e-05,
      "logits/chosen": -2.138289213180542,
      "logits/rejected": -2.1273224353790283,
      "logps/chosen": -173.62033081054688,
      "logps/rejected": -163.26397705078125,
      "loss": 0.8012,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.24764633178710938,
      "rewards/margins": -0.08084635436534882,
      "rewards/rejected": -0.16679996252059937,
      "step": 598
    },
    {
      "epoch": 0.78,
      "learning_rate": 4.3874528721721624e-05,
      "logits/chosen": -2.150501012802124,
      "logits/rejected": -2.1304874420166016,
      "logps/chosen": -167.4502716064453,
      "logps/rejected": -207.66793823242188,
      "loss": 0.859,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3565550744533539,
      "rewards/margins": -0.1929093897342682,
      "rewards/rejected": -0.1636456847190857,
      "step": 599
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.385101381140633e-05,
      "logits/chosen": -2.0221786499023438,
      "logits/rejected": -2.1972155570983887,
      "logps/chosen": -174.52090454101562,
      "logps/rejected": -189.63348388671875,
      "loss": 0.6853,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1927257478237152,
      "rewards/margins": 0.07324501872062683,
      "rewards/rejected": -0.26597073674201965,
      "step": 600
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.382746017949203e-05,
      "logits/chosen": -2.0691418647766113,
      "logits/rejected": -2.0149199962615967,
      "logps/chosen": -247.23388671875,
      "logps/rejected": -234.80191040039062,
      "loss": 0.8027,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.552363395690918,
      "rewards/margins": -0.18797653913497925,
      "rewards/rejected": -0.36438679695129395,
      "step": 601
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.380386787435992e-05,
      "logits/chosen": -2.1596169471740723,
      "logits/rejected": -2.1461353302001953,
      "logps/chosen": -171.2084503173828,
      "logps/rejected": -213.65097045898438,
      "loss": 0.7502,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.5347078442573547,
      "rewards/margins": -0.039867326617240906,
      "rewards/rejected": -0.49484050273895264,
      "step": 602
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.378023694447061e-05,
      "logits/chosen": -2.121032953262329,
      "logits/rejected": -2.1503822803497314,
      "logps/chosen": -164.81271362304688,
      "logps/rejected": -170.76075744628906,
      "loss": 0.6693,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.01932806894183159,
      "rewards/margins": 0.08725270628929138,
      "rewards/rejected": -0.06792464852333069,
      "step": 603
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.375656743836407e-05,
      "logits/chosen": -2.059199094772339,
      "logits/rejected": -2.071756601333618,
      "logps/chosen": -164.81121826171875,
      "logps/rejected": -161.324951171875,
      "loss": 0.6746,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.22563201189041138,
      "rewards/margins": 0.145117849111557,
      "rewards/rejected": -0.37074992060661316,
      "step": 604
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.373285940465948e-05,
      "logits/chosen": -2.096938371658325,
      "logits/rejected": -2.2146358489990234,
      "logps/chosen": -149.6543731689453,
      "logps/rejected": -172.9806671142578,
      "loss": 0.7894,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2511146366596222,
      "rewards/margins": 0.0032073892652988434,
      "rewards/rejected": -0.2543220520019531,
      "step": 605
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.370911289205518e-05,
      "logits/chosen": -2.0443522930145264,
      "logits/rejected": -2.012732982635498,
      "logps/chosen": -181.68507385253906,
      "logps/rejected": -182.77114868164062,
      "loss": 0.6579,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.19644834101200104,
      "rewards/margins": 0.14906372129917145,
      "rewards/rejected": -0.3455120623111725,
      "step": 606
    },
    {
      "epoch": 0.79,
      "learning_rate": 4.368532794932854e-05,
      "logits/chosen": -2.0499818325042725,
      "logits/rejected": -2.0729148387908936,
      "logps/chosen": -156.1490020751953,
      "logps/rejected": -156.59902954101562,
      "loss": 0.7566,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.12633728981018066,
      "rewards/margins": -0.09627077728509903,
      "rewards/rejected": -0.030066512525081635,
      "step": 607
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.366150462533588e-05,
      "logits/chosen": -2.1601145267486572,
      "logits/rejected": -2.153270959854126,
      "logps/chosen": -130.52975463867188,
      "logps/rejected": -131.89930725097656,
      "loss": 0.8018,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.30306872725486755,
      "rewards/margins": -0.10493332147598267,
      "rewards/rejected": -0.19813542068004608,
      "step": 608
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.363764296901234e-05,
      "logits/chosen": -2.1186208724975586,
      "logits/rejected": -2.134394645690918,
      "logps/chosen": -183.10049438476562,
      "logps/rejected": -175.95849609375,
      "loss": 0.8474,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.47242993116378784,
      "rewards/margins": -0.21657848358154297,
      "rewards/rejected": -0.25585147738456726,
      "step": 609
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.361374302937182e-05,
      "logits/chosen": -2.0684666633605957,
      "logits/rejected": -2.0428881645202637,
      "logps/chosen": -207.9459686279297,
      "logps/rejected": -229.56698608398438,
      "loss": 0.8513,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.3699374198913574,
      "rewards/margins": -0.2574125826358795,
      "rewards/rejected": -0.11252487450838089,
      "step": 610
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.358980485550683e-05,
      "logits/chosen": -1.949844479560852,
      "logits/rejected": -1.9453861713409424,
      "logps/chosen": -143.38499450683594,
      "logps/rejected": -150.81947326660156,
      "loss": 0.705,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5936511754989624,
      "rewards/margins": 0.01353779248893261,
      "rewards/rejected": -0.6071889400482178,
      "step": 611
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.356582849658845e-05,
      "logits/chosen": -1.9948573112487793,
      "logits/rejected": -2.0161683559417725,
      "logps/chosen": -140.34910583496094,
      "logps/rejected": -141.26271057128906,
      "loss": 0.5736,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19039779901504517,
      "rewards/margins": 0.31717830896377563,
      "rewards/rejected": -0.5075761079788208,
      "step": 612
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.354181400186617e-05,
      "logits/chosen": -1.9521052837371826,
      "logits/rejected": -1.9075651168823242,
      "logps/chosen": -137.3512420654297,
      "logps/rejected": -136.6510467529297,
      "loss": 0.7616,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.45448195934295654,
      "rewards/margins": -0.034489963203668594,
      "rewards/rejected": -0.41999197006225586,
      "step": 613
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.351776142066782e-05,
      "logits/chosen": -2.138842821121216,
      "logits/rejected": -2.1373250484466553,
      "logps/chosen": -193.3833465576172,
      "logps/rejected": -193.51292419433594,
      "loss": 0.6409,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2606329023838043,
      "rewards/margins": 0.1631646752357483,
      "rewards/rejected": -0.423797607421875,
      "step": 614
    },
    {
      "epoch": 0.8,
      "learning_rate": 4.349367080239946e-05,
      "logits/chosen": -2.118267774581909,
      "logits/rejected": -2.135613441467285,
      "logps/chosen": -250.55662536621094,
      "logps/rejected": -243.0400390625,
      "loss": 0.7276,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.7016344666481018,
      "rewards/margins": -0.04274986684322357,
      "rewards/rejected": -0.6588846445083618,
      "step": 615
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.34695421965453e-05,
      "logits/chosen": -2.00390362739563,
      "logits/rejected": -1.9568325281143188,
      "logps/chosen": -203.2637176513672,
      "logps/rejected": -215.809326171875,
      "loss": 0.7331,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3516182005405426,
      "rewards/margins": -0.01222594827413559,
      "rewards/rejected": -0.3393922746181488,
      "step": 616
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.344537565266755e-05,
      "logits/chosen": -2.201554536819458,
      "logits/rejected": -2.2679710388183594,
      "logps/chosen": -168.6824188232422,
      "logps/rejected": -193.9791259765625,
      "loss": 0.6135,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3374904990196228,
      "rewards/margins": 0.24233487248420715,
      "rewards/rejected": -0.5798253417015076,
      "step": 617
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.342117122040637e-05,
      "logits/chosen": -1.8600742816925049,
      "logits/rejected": -2.029170274734497,
      "logps/chosen": -151.89068603515625,
      "logps/rejected": -183.169189453125,
      "loss": 0.5888,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1524229496717453,
      "rewards/margins": 0.2849957346916199,
      "rewards/rejected": -0.43741869926452637,
      "step": 618
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.339692894947974e-05,
      "logits/chosen": -2.1574344635009766,
      "logits/rejected": -2.1366348266601562,
      "logps/chosen": -168.2608642578125,
      "logps/rejected": -169.64785766601562,
      "loss": 0.748,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.5251599550247192,
      "rewards/margins": -0.011976517736911774,
      "rewards/rejected": -0.5131835341453552,
      "step": 619
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.3372648889683364e-05,
      "logits/chosen": -2.0989270210266113,
      "logits/rejected": -2.053802490234375,
      "logps/chosen": -147.33477783203125,
      "logps/rejected": -148.05438232421875,
      "loss": 0.6681,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.18421587347984314,
      "rewards/margins": 0.19642779231071472,
      "rewards/rejected": -0.38064366579055786,
      "step": 620
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.334833109089057e-05,
      "logits/chosen": -2.010010242462158,
      "logits/rejected": -2.007097005844116,
      "logps/chosen": -162.26063537597656,
      "logps/rejected": -156.50814819335938,
      "loss": 0.6747,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4700685143470764,
      "rewards/margins": 0.10477405786514282,
      "rewards/rejected": -0.5748425126075745,
      "step": 621
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.33239756030522e-05,
      "logits/chosen": -1.7842457294464111,
      "logits/rejected": -1.7858459949493408,
      "logps/chosen": -133.84458923339844,
      "logps/rejected": -175.0372314453125,
      "loss": 0.6192,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.2971382737159729,
      "rewards/margins": 0.2653331458568573,
      "rewards/rejected": -0.5624713897705078,
      "step": 622
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.329958247619651e-05,
      "logits/chosen": -2.1888294219970703,
      "logits/rejected": -2.1918630599975586,
      "logps/chosen": -153.9444580078125,
      "logps/rejected": -150.8408966064453,
      "loss": 0.6443,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15923231840133667,
      "rewards/margins": 0.16449183225631714,
      "rewards/rejected": -0.3237241506576538,
      "step": 623
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.3275151760429075e-05,
      "logits/chosen": -2.0962228775024414,
      "logits/rejected": -2.1192712783813477,
      "logps/chosen": -155.5863494873047,
      "logps/rejected": -150.6765594482422,
      "loss": 0.5493,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19339366257190704,
      "rewards/margins": 0.4053744971752167,
      "rewards/rejected": -0.5987681150436401,
      "step": 624
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.325068350593268e-05,
      "logits/chosen": -2.095531702041626,
      "logits/rejected": -2.180366039276123,
      "logps/chosen": -187.6971893310547,
      "logps/rejected": -229.3963165283203,
      "loss": 0.7277,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.45450133085250854,
      "rewards/margins": 0.05496390163898468,
      "rewards/rejected": -0.509465217590332,
      "step": 625
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.322617776296723e-05,
      "logits/chosen": -2.116663694381714,
      "logits/rejected": -2.131500244140625,
      "logps/chosen": -147.40220642089844,
      "logps/rejected": -160.78799438476562,
      "loss": 0.7796,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.3981628715991974,
      "rewards/margins": -0.08542328327894211,
      "rewards/rejected": -0.3127395808696747,
      "step": 626
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.320163458186961e-05,
      "logits/chosen": -1.8610426187515259,
      "logits/rejected": -1.8280242681503296,
      "logps/chosen": -142.5250244140625,
      "logps/rejected": -167.79307556152344,
      "loss": 0.6001,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.26573097705841064,
      "rewards/margins": 0.3146597445011139,
      "rewards/rejected": -0.5803906917572021,
      "step": 627
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.317705401305362e-05,
      "logits/chosen": -2.2919328212738037,
      "logits/rejected": -2.295372486114502,
      "logps/chosen": -182.9703369140625,
      "logps/rejected": -184.76123046875,
      "loss": 0.6323,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2809330224990845,
      "rewards/margins": 0.18110422790050507,
      "rewards/rejected": -0.46203726530075073,
      "step": 628
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.315243610700986e-05,
      "logits/chosen": -1.962925910949707,
      "logits/rejected": -1.932532787322998,
      "logps/chosen": -135.61175537109375,
      "logps/rejected": -145.57708740234375,
      "loss": 0.5795,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2901972532272339,
      "rewards/margins": 0.3114909827709198,
      "rewards/rejected": -0.6016882658004761,
      "step": 629
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.312778091430563e-05,
      "logits/chosen": -2.0856752395629883,
      "logits/rejected": -2.060380697250366,
      "logps/chosen": -161.23788452148438,
      "logps/rejected": -174.86415100097656,
      "loss": 0.685,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3572569191455841,
      "rewards/margins": 0.08018150925636292,
      "rewards/rejected": -0.437438428401947,
      "step": 630
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.310308848558479e-05,
      "logits/chosen": -1.967565894126892,
      "logits/rejected": -2.0015602111816406,
      "logps/chosen": -158.85549926757812,
      "logps/rejected": -165.99868774414062,
      "loss": 0.643,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.298388808965683,
      "rewards/margins": 0.15071076154708862,
      "rewards/rejected": -0.4490995407104492,
      "step": 631
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.3078358871567706e-05,
      "logits/chosen": -1.9660686254501343,
      "logits/rejected": -2.092501163482666,
      "logps/chosen": -134.13931274414062,
      "logps/rejected": -151.00088500976562,
      "loss": 0.6324,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.28502339124679565,
      "rewards/margins": 0.22240550816059113,
      "rewards/rejected": -0.5074288845062256,
      "step": 632
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.305359212305115e-05,
      "logits/chosen": -2.0704879760742188,
      "logits/rejected": -2.108783721923828,
      "logps/chosen": -151.75942993164062,
      "logps/rejected": -169.45274353027344,
      "loss": 0.6533,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2645598351955414,
      "rewards/margins": 0.18161676824092865,
      "rewards/rejected": -0.44617658853530884,
      "step": 633
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.302878829090813e-05,
      "logits/chosen": -2.2878875732421875,
      "logits/rejected": -2.275372266769409,
      "logps/chosen": -171.14463806152344,
      "logps/rejected": -161.89524841308594,
      "loss": 0.721,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.572907567024231,
      "rewards/margins": 0.08118201792240143,
      "rewards/rejected": -0.6540895700454712,
      "step": 634
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.300394742608784e-05,
      "logits/chosen": -2.080106019973755,
      "logits/rejected": -2.0426576137542725,
      "logps/chosen": -151.9457550048828,
      "logps/rejected": -161.84259033203125,
      "loss": 0.6613,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22103726863861084,
      "rewards/margins": 0.21995976567268372,
      "rewards/rejected": -0.44099700450897217,
      "step": 635
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2979069579615564e-05,
      "logits/chosen": -2.027768850326538,
      "logits/rejected": -2.0777463912963867,
      "logps/chosen": -177.09764099121094,
      "logps/rejected": -151.2614288330078,
      "loss": 0.8068,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6577314734458923,
      "rewards/margins": -0.037055857479572296,
      "rewards/rejected": -0.6206756234169006,
      "step": 636
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.2954154802592514e-05,
      "logits/chosen": -2.1159207820892334,
      "logits/rejected": -2.081036329269409,
      "logps/chosen": -177.8500213623047,
      "logps/rejected": -154.96888732910156,
      "loss": 0.8062,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.4962155818939209,
      "rewards/margins": -0.15886664390563965,
      "rewards/rejected": -0.33734893798828125,
      "step": 637
    },
    {
      "epoch": 0.83,
      "learning_rate": 4.292920314619578e-05,
      "logits/chosen": -2.139322519302368,
      "logits/rejected": -2.1055707931518555,
      "logps/chosen": -135.03768920898438,
      "logps/rejected": -146.54347229003906,
      "loss": 0.7386,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.4281715750694275,
      "rewards/margins": 0.03442011773586273,
      "rewards/rejected": -0.4625917077064514,
      "step": 638
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.290421466167822e-05,
      "logits/chosen": -2.173722743988037,
      "logits/rejected": -2.145287036895752,
      "logps/chosen": -188.748779296875,
      "logps/rejected": -167.95037841796875,
      "loss": 0.7986,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.45367497205734253,
      "rewards/margins": -0.16028070449829102,
      "rewards/rejected": -0.2933943271636963,
      "step": 639
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.2879189400368314e-05,
      "logits/chosen": -1.828595757484436,
      "logits/rejected": -1.8133007287979126,
      "logps/chosen": -171.7655792236328,
      "logps/rejected": -181.7562255859375,
      "loss": 0.637,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15220165252685547,
      "rewards/margins": 0.2077586054801941,
      "rewards/rejected": -0.35996025800704956,
      "step": 640
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.2854127413670096e-05,
      "logits/chosen": -2.1601057052612305,
      "logits/rejected": -2.1688709259033203,
      "logps/chosen": -140.67388916015625,
      "logps/rejected": -159.5793914794922,
      "loss": 0.6556,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.19438128173351288,
      "rewards/margins": 0.14893627166748047,
      "rewards/rejected": -0.34331756830215454,
      "step": 641
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.282902875306304e-05,
      "logits/chosen": -2.2678325176239014,
      "logits/rejected": -2.235651969909668,
      "logps/chosen": -206.64051818847656,
      "logps/rejected": -193.34620666503906,
      "loss": 0.7065,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3568398952484131,
      "rewards/margins": 0.062468186020851135,
      "rewards/rejected": -0.419308066368103,
      "step": 642
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.280389347010194e-05,
      "logits/chosen": -2.0710811614990234,
      "logits/rejected": -2.140387535095215,
      "logps/chosen": -148.68856811523438,
      "logps/rejected": -162.29550170898438,
      "loss": 0.7149,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.37908732891082764,
      "rewards/margins": 0.052500005811452866,
      "rewards/rejected": -0.4315873384475708,
      "step": 643
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.277872161641682e-05,
      "logits/chosen": -2.075469493865967,
      "logits/rejected": -2.0515270233154297,
      "logps/chosen": -154.94790649414062,
      "logps/rejected": -143.33999633789062,
      "loss": 0.8673,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.636205792427063,
      "rewards/margins": -0.23466622829437256,
      "rewards/rejected": -0.40153956413269043,
      "step": 644
    },
    {
      "epoch": 0.84,
      "learning_rate": 4.275351324371283e-05,
      "logits/chosen": -2.034024953842163,
      "logits/rejected": -2.0730140209198,
      "logps/chosen": -165.8710479736328,
      "logps/rejected": -167.48043823242188,
      "loss": 0.6052,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.4217650890350342,
      "rewards/margins": 0.24776552617549896,
      "rewards/rejected": -0.6695306301116943,
      "step": 645
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.2728268403770145e-05,
      "logits/chosen": -2.018975019454956,
      "logits/rejected": -2.0665860176086426,
      "logps/chosen": -168.4711456298828,
      "logps/rejected": -167.21612548828125,
      "loss": 0.719,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.43721261620521545,
      "rewards/margins": 0.02011800743639469,
      "rewards/rejected": -0.4573306143283844,
      "step": 646
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.270298714844381e-05,
      "logits/chosen": -2.073598623275757,
      "logits/rejected": -2.032815456390381,
      "logps/chosen": -168.9475860595703,
      "logps/rejected": -157.1396942138672,
      "loss": 0.7293,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.24852538108825684,
      "rewards/margins": -0.01923494040966034,
      "rewards/rejected": -0.2292904555797577,
      "step": 647
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.267766952966369e-05,
      "logits/chosen": -1.8729684352874756,
      "logits/rejected": -1.8412991762161255,
      "logps/chosen": -157.169677734375,
      "logps/rejected": -138.6353302001953,
      "loss": 0.7324,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.472010999917984,
      "rewards/margins": 0.06063162162899971,
      "rewards/rejected": -0.5326426029205322,
      "step": 648
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.2652315599434354e-05,
      "logits/chosen": -1.9037437438964844,
      "logits/rejected": -1.9179785251617432,
      "logps/chosen": -149.9954071044922,
      "logps/rejected": -150.65032958984375,
      "loss": 0.7982,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.272285133600235,
      "rewards/margins": -0.07758139818906784,
      "rewards/rejected": -0.19470372796058655,
      "step": 649
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.262692540983496e-05,
      "logits/chosen": -2.205641746520996,
      "logits/rejected": -2.2717151641845703,
      "logps/chosen": -150.75364685058594,
      "logps/rejected": -156.90733337402344,
      "loss": 0.8112,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3120537996292114,
      "rewards/margins": -0.1661502569913864,
      "rewards/rejected": -0.1459035575389862,
      "step": 650
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.2601499013019126e-05,
      "logits/chosen": -1.9804850816726685,
      "logits/rejected": -2.0665392875671387,
      "logps/chosen": -148.04840087890625,
      "logps/rejected": -172.39447021484375,
      "loss": 0.6469,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.41176366806030273,
      "rewards/margins": 0.12474896013736725,
      "rewards/rejected": -0.5365126132965088,
      "step": 651
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.257603646121484e-05,
      "logits/chosen": -1.8488759994506836,
      "logits/rejected": -1.9345241785049438,
      "logps/chosen": -150.47325134277344,
      "logps/rejected": -180.80213928222656,
      "loss": 0.7586,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2809077203273773,
      "rewards/margins": -0.0009739026427268982,
      "rewards/rejected": -0.2799338400363922,
      "step": 652
    },
    {
      "epoch": 0.85,
      "learning_rate": 4.2550537806724384e-05,
      "logits/chosen": -2.0807509422302246,
      "logits/rejected": -2.0452427864074707,
      "logps/chosen": -152.01800537109375,
      "logps/rejected": -177.10731506347656,
      "loss": 0.6403,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19310708343982697,
      "rewards/margins": 0.18130317330360413,
      "rewards/rejected": -0.3744102120399475,
      "step": 653
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.2525003101924164e-05,
      "logits/chosen": -1.9181900024414062,
      "logits/rejected": -1.9261229038238525,
      "logps/chosen": -140.71975708007812,
      "logps/rejected": -151.97988891601562,
      "loss": 0.6749,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.08997268974781036,
      "rewards/margins": 0.10172852873802185,
      "rewards/rejected": -0.1917012333869934,
      "step": 654
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.249943239926467e-05,
      "logits/chosen": -2.0650923252105713,
      "logits/rejected": -1.9644784927368164,
      "logps/chosen": -176.4941864013672,
      "logps/rejected": -159.2071533203125,
      "loss": 0.7129,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.6318143010139465,
      "rewards/margins": 0.05239555612206459,
      "rewards/rejected": -0.6842098832130432,
      "step": 655
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.247382575127031e-05,
      "logits/chosen": -2.0065248012542725,
      "logits/rejected": -2.021730661392212,
      "logps/chosen": -215.60427856445312,
      "logps/rejected": -202.29327392578125,
      "loss": 0.6533,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.05739132687449455,
      "rewards/margins": 0.12948471307754517,
      "rewards/rejected": -0.18687602877616882,
      "step": 656
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.2448183210539334e-05,
      "logits/chosen": -2.0068016052246094,
      "logits/rejected": -1.9955005645751953,
      "logps/chosen": -178.9858856201172,
      "logps/rejected": -138.95423889160156,
      "loss": 0.7054,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005308074876666069,
      "rewards/margins": 0.04537278786301613,
      "rewards/rejected": -0.040064722299575806,
      "step": 657
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.2422504829743724e-05,
      "logits/chosen": -2.07608699798584,
      "logits/rejected": -2.0257248878479004,
      "logps/chosen": -175.4975128173828,
      "logps/rejected": -152.2142333984375,
      "loss": 0.6871,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.006427427753806114,
      "rewards/margins": 0.06741438060998917,
      "rewards/rejected": -0.07384180277585983,
      "step": 658
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.239679066162907e-05,
      "logits/chosen": -1.834554672241211,
      "logits/rejected": -1.9117746353149414,
      "logps/chosen": -152.05563354492188,
      "logps/rejected": -168.4638671875,
      "loss": 0.6393,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.16257554292678833,
      "rewards/margins": 0.14442546665668488,
      "rewards/rejected": -0.3070010244846344,
      "step": 659
    },
    {
      "epoch": 0.86,
      "learning_rate": 4.237104075901449e-05,
      "logits/chosen": -1.856727123260498,
      "logits/rejected": -1.8436062335968018,
      "logps/chosen": -166.12420654296875,
      "logps/rejected": -169.3896484375,
      "loss": 0.8167,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.6063433289527893,
      "rewards/margins": -0.16782012581825256,
      "rewards/rejected": -0.4385232627391815,
      "step": 660
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.234525517479248e-05,
      "logits/chosen": -1.8753873109817505,
      "logits/rejected": -1.8495755195617676,
      "logps/chosen": -149.54005432128906,
      "logps/rejected": -163.44752502441406,
      "loss": 0.5914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15266531705856323,
      "rewards/margins": 0.2658059000968933,
      "rewards/rejected": -0.41847115755081177,
      "step": 661
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.2319433961928844e-05,
      "logits/chosen": -2.13688063621521,
      "logits/rejected": -2.1577110290527344,
      "logps/chosen": -189.70277404785156,
      "logps/rejected": -197.83218383789062,
      "loss": 0.7608,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2501004934310913,
      "rewards/margins": -0.03931330889463425,
      "rewards/rejected": -0.21078716218471527,
      "step": 662
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.229357717346257e-05,
      "logits/chosen": -1.8148566484451294,
      "logits/rejected": -1.7939611673355103,
      "logps/chosen": -164.1804656982422,
      "logps/rejected": -168.58926391601562,
      "loss": 0.7507,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.46129778027534485,
      "rewards/margins": -0.07121908664703369,
      "rewards/rejected": -0.39007866382598877,
      "step": 663
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.226768486250572e-05,
      "logits/chosen": -2.0947425365448,
      "logits/rejected": -2.138808250427246,
      "logps/chosen": -150.54244995117188,
      "logps/rejected": -168.19464111328125,
      "loss": 0.6282,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.030175644904375076,
      "rewards/margins": 0.19879963994026184,
      "rewards/rejected": -0.2289752960205078,
      "step": 664
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.224175708224332e-05,
      "logits/chosen": -2.044949769973755,
      "logits/rejected": -2.0898711681365967,
      "logps/chosen": -167.0792999267578,
      "logps/rejected": -173.0589599609375,
      "loss": 0.7429,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.28365832567214966,
      "rewards/margins": -0.027682170271873474,
      "rewards/rejected": -0.255976140499115,
      "step": 665
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.221579388593326e-05,
      "logits/chosen": -2.167201042175293,
      "logits/rejected": -2.199467658996582,
      "logps/chosen": -139.47561645507812,
      "logps/rejected": -147.73927307128906,
      "loss": 0.6791,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2867812514305115,
      "rewards/margins": 0.12948867678642273,
      "rewards/rejected": -0.4162698984146118,
      "step": 666
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.218979532690616e-05,
      "logits/chosen": -2.078275680541992,
      "logits/rejected": -2.125436305999756,
      "logps/chosen": -132.52284240722656,
      "logps/rejected": -142.66737365722656,
      "loss": 0.9048,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.31447887420654297,
      "rewards/margins": -0.27543795108795166,
      "rewards/rejected": -0.0390409454703331,
      "step": 667
    },
    {
      "epoch": 0.87,
      "learning_rate": 4.216376145856529e-05,
      "logits/chosen": -1.9571113586425781,
      "logits/rejected": -1.9044986963272095,
      "logps/chosen": -149.46768188476562,
      "logps/rejected": -135.63412475585938,
      "loss": 0.5922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15536966919898987,
      "rewards/margins": 0.27842316031455994,
      "rewards/rejected": -0.4337928295135498,
      "step": 668
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.213769233438646e-05,
      "logits/chosen": -2.0616254806518555,
      "logits/rejected": -2.054438352584839,
      "logps/chosen": -171.1319122314453,
      "logps/rejected": -173.5122528076172,
      "loss": 0.8061,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3872271478176117,
      "rewards/margins": -0.13944876194000244,
      "rewards/rejected": -0.24777841567993164,
      "step": 669
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.211158800791788e-05,
      "logits/chosen": -2.0811805725097656,
      "logits/rejected": -2.0651638507843018,
      "logps/chosen": -134.8317413330078,
      "logps/rejected": -159.145263671875,
      "loss": 0.6551,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.16378769278526306,
      "rewards/margins": 0.17533215880393982,
      "rewards/rejected": -0.33911988139152527,
      "step": 670
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.208544853278008e-05,
      "logits/chosen": -1.8314135074615479,
      "logits/rejected": -1.7468427419662476,
      "logps/chosen": -162.07394409179688,
      "logps/rejected": -195.4473876953125,
      "loss": 0.9136,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.5376362800598145,
      "rewards/margins": -0.3255177438259125,
      "rewards/rejected": -0.21211856603622437,
      "step": 671
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.205927396266577e-05,
      "logits/chosen": -2.237910270690918,
      "logits/rejected": -2.2668380737304688,
      "logps/chosen": -190.4712677001953,
      "logps/rejected": -189.41978454589844,
      "loss": 0.7084,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.310894250869751,
      "rewards/margins": 0.13236525654792786,
      "rewards/rejected": -0.44325950741767883,
      "step": 672
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.203306435133978e-05,
      "logits/chosen": -2.1241772174835205,
      "logits/rejected": -2.1544597148895264,
      "logps/chosen": -197.74461364746094,
      "logps/rejected": -223.18495178222656,
      "loss": 0.7411,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5014972686767578,
      "rewards/margins": 0.0350608304142952,
      "rewards/rejected": -0.5365581512451172,
      "step": 673
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.200681975263888e-05,
      "logits/chosen": -1.9514243602752686,
      "logits/rejected": -1.9178723096847534,
      "logps/chosen": -148.33969116210938,
      "logps/rejected": -155.63035583496094,
      "loss": 0.8101,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.4579694867134094,
      "rewards/margins": -0.15268799662590027,
      "rewards/rejected": -0.30528151988983154,
      "step": 674
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.1980540220471744e-05,
      "logits/chosen": -2.1476829051971436,
      "logits/rejected": -2.1371512413024902,
      "logps/chosen": -148.34912109375,
      "logps/rejected": -143.85255432128906,
      "loss": 0.7207,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2867552638053894,
      "rewards/margins": 0.0834084004163742,
      "rewards/rejected": -0.3701636791229248,
      "step": 675
    },
    {
      "epoch": 0.88,
      "learning_rate": 4.195422580881878e-05,
      "logits/chosen": -2.039616346359253,
      "logits/rejected": -2.122281551361084,
      "logps/chosen": -146.25778198242188,
      "logps/rejected": -152.91786193847656,
      "loss": 0.6984,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.27001526951789856,
      "rewards/margins": 0.08569072186946869,
      "rewards/rejected": -0.35570600628852844,
      "step": 676
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.192787657173204e-05,
      "logits/chosen": -2.0761122703552246,
      "logits/rejected": -2.03676438331604,
      "logps/chosen": -260.5777587890625,
      "logps/rejected": -249.19309997558594,
      "loss": 0.7267,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06880796700716019,
      "rewards/margins": -0.00840199738740921,
      "rewards/rejected": -0.06040596961975098,
      "step": 677
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.1901492563335115e-05,
      "logits/chosen": -1.779012680053711,
      "logits/rejected": -1.868876576423645,
      "logps/chosen": -171.57522583007812,
      "logps/rejected": -179.360107421875,
      "loss": 0.7206,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.45824557542800903,
      "rewards/margins": 0.07630608975887299,
      "rewards/rejected": -0.534551739692688,
      "step": 678
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.187507383782303e-05,
      "logits/chosen": -1.95026695728302,
      "logits/rejected": -2.003551959991455,
      "logps/chosen": -152.73373413085938,
      "logps/rejected": -153.47882080078125,
      "loss": 0.742,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.15587645769119263,
      "rewards/margins": 0.019024118781089783,
      "rewards/rejected": -0.1749005764722824,
      "step": 679
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.1848620449462115e-05,
      "logits/chosen": -2.1446237564086914,
      "logits/rejected": -2.1463828086853027,
      "logps/chosen": -155.42047119140625,
      "logps/rejected": -152.02236938476562,
      "loss": 0.7138,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.29967939853668213,
      "rewards/margins": 0.058842338621616364,
      "rewards/rejected": -0.3585217297077179,
      "step": 680
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.1822132452589885e-05,
      "logits/chosen": -1.958126425743103,
      "logits/rejected": -1.9775969982147217,
      "logps/chosen": -147.2157745361328,
      "logps/rejected": -177.09339904785156,
      "loss": 0.7635,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.3013852834701538,
      "rewards/margins": -0.06730309128761292,
      "rewards/rejected": -0.2340822070837021,
      "step": 681
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.1795609901614966e-05,
      "logits/chosen": -2.1244254112243652,
      "logits/rejected": -2.1423747539520264,
      "logps/chosen": -154.98565673828125,
      "logps/rejected": -160.26119995117188,
      "loss": 0.7795,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2978401184082031,
      "rewards/margins": -0.14463642239570618,
      "rewards/rejected": -0.15320369601249695,
      "step": 682
    },
    {
      "epoch": 0.89,
      "learning_rate": 4.176905285101695e-05,
      "logits/chosen": -2.1043314933776855,
      "logits/rejected": -2.0934536457061768,
      "logps/chosen": -168.54006958007812,
      "logps/rejected": -175.0818634033203,
      "loss": 0.7897,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6096583008766174,
      "rewards/margins": -0.020460113883018494,
      "rewards/rejected": -0.5891982316970825,
      "step": 683
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.17424613553463e-05,
      "logits/chosen": -2.0135533809661865,
      "logits/rejected": -2.057865858078003,
      "logps/chosen": -165.50119018554688,
      "logps/rejected": -158.755859375,
      "loss": 0.7246,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.37735715508461,
      "rewards/margins": -0.000978667289018631,
      "rewards/rejected": -0.37637850642204285,
      "step": 684
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.171583546922423e-05,
      "logits/chosen": -1.9394524097442627,
      "logits/rejected": -1.9916284084320068,
      "logps/chosen": -149.19032287597656,
      "logps/rejected": -168.09466552734375,
      "loss": 0.7205,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.17173324525356293,
      "rewards/margins": 0.10954613983631134,
      "rewards/rejected": -0.2812793552875519,
      "step": 685
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.1689175247342584e-05,
      "logits/chosen": -2.00577712059021,
      "logits/rejected": -1.9910097122192383,
      "logps/chosen": -181.58145141601562,
      "logps/rejected": -168.53257751464844,
      "loss": 0.7636,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5042218565940857,
      "rewards/margins": -0.05149242654442787,
      "rewards/rejected": -0.4527294337749481,
      "step": 686
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.1662480744463744e-05,
      "logits/chosen": -1.9668911695480347,
      "logits/rejected": -2.0537397861480713,
      "logps/chosen": -146.09317016601562,
      "logps/rejected": -173.7940673828125,
      "loss": 0.5798,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.5452170968055725,
      "rewards/margins": 0.28624922037124634,
      "rewards/rejected": -0.8314663767814636,
      "step": 687
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.163575201542052e-05,
      "logits/chosen": -1.8841594457626343,
      "logits/rejected": -1.8778733015060425,
      "logps/chosen": -161.64395141601562,
      "logps/rejected": -167.02130126953125,
      "loss": 0.6612,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.2581769526004791,
      "rewards/margins": 0.09805278480052948,
      "rewards/rejected": -0.3562297224998474,
      "step": 688
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.1608989115116e-05,
      "logits/chosen": -2.224517583847046,
      "logits/rejected": -2.1719701290130615,
      "logps/chosen": -157.36236572265625,
      "logps/rejected": -159.3743438720703,
      "loss": 0.7116,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.5532806515693665,
      "rewards/margins": 0.07318402081727982,
      "rewards/rejected": -0.6264646649360657,
      "step": 689
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.158219209852349e-05,
      "logits/chosen": -2.2228987216949463,
      "logits/rejected": -2.118849277496338,
      "logps/chosen": -157.40219116210938,
      "logps/rejected": -158.98670959472656,
      "loss": 0.769,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.47785037755966187,
      "rewards/margins": -0.026296239346265793,
      "rewards/rejected": -0.45155417919158936,
      "step": 690
    },
    {
      "epoch": 0.9,
      "learning_rate": 4.155536102068636e-05,
      "logits/chosen": -2.170536756515503,
      "logits/rejected": -2.1680848598480225,
      "logps/chosen": -187.9390869140625,
      "logps/rejected": -193.16246032714844,
      "loss": 0.7623,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00898093730211258,
      "rewards/margins": -0.09791917353868484,
      "rewards/rejected": 0.10690011084079742,
      "step": 691
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.152849593671793e-05,
      "logits/chosen": -2.271533727645874,
      "logits/rejected": -2.3052127361297607,
      "logps/chosen": -158.00819396972656,
      "logps/rejected": -174.9010467529297,
      "loss": 0.8109,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.4787377119064331,
      "rewards/margins": -0.16133984923362732,
      "rewards/rejected": -0.3173978626728058,
      "step": 692
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.1501596901801384e-05,
      "logits/chosen": -2.1939315795898438,
      "logits/rejected": -2.177248954772949,
      "logps/chosen": -142.37559509277344,
      "logps/rejected": -153.59266662597656,
      "loss": 0.7123,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.2645539939403534,
      "rewards/margins": 0.06362085044384003,
      "rewards/rejected": -0.3281748294830322,
      "step": 693
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.147466397118968e-05,
      "logits/chosen": -2.2683045864105225,
      "logits/rejected": -2.251051187515259,
      "logps/chosen": -157.3998565673828,
      "logps/rejected": -158.64883422851562,
      "loss": 0.7921,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.3830302655696869,
      "rewards/margins": -0.15283122658729553,
      "rewards/rejected": -0.23019905388355255,
      "step": 694
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.144769720020533e-05,
      "logits/chosen": -2.172579050064087,
      "logits/rejected": -2.182398796081543,
      "logps/chosen": -166.13075256347656,
      "logps/rejected": -157.43788146972656,
      "loss": 0.8064,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.5050195455551147,
      "rewards/margins": -0.07979856431484222,
      "rewards/rejected": -0.42522096633911133,
      "step": 695
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.142069664424041e-05,
      "logits/chosen": -2.117938280105591,
      "logits/rejected": -2.1075210571289062,
      "logps/chosen": -161.34828186035156,
      "logps/rejected": -152.29771423339844,
      "loss": 0.7732,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.18459086120128632,
      "rewards/margins": -0.009315093979239464,
      "rewards/rejected": -0.1752757579088211,
      "step": 696
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.139366235875637e-05,
      "logits/chosen": -1.6788434982299805,
      "logits/rejected": -1.7080583572387695,
      "logps/chosen": -161.94412231445312,
      "logps/rejected": -151.36595153808594,
      "loss": 0.6239,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19800153374671936,
      "rewards/margins": 0.2967408299446106,
      "rewards/rejected": -0.49474233388900757,
      "step": 697
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.136659439928397e-05,
      "logits/chosen": -1.9581472873687744,
      "logits/rejected": -2.0028040409088135,
      "logps/chosen": -155.76913452148438,
      "logps/rejected": -160.2482452392578,
      "loss": 0.7087,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.3814413845539093,
      "rewards/margins": 0.04772702604532242,
      "rewards/rejected": -0.4291684031486511,
      "step": 698
    },
    {
      "epoch": 0.91,
      "learning_rate": 4.13394928214231e-05,
      "logits/chosen": -2.0003318786621094,
      "logits/rejected": -2.0119810104370117,
      "logps/chosen": -163.40011596679688,
      "logps/rejected": -171.38818359375,
      "loss": 0.6004,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04564451798796654,
      "rewards/margins": 0.31329357624053955,
      "rewards/rejected": -0.3589380979537964,
      "step": 699
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.1312357680842735e-05,
      "logits/chosen": -2.158578395843506,
      "logits/rejected": -2.1833291053771973,
      "logps/chosen": -147.8467254638672,
      "logps/rejected": -146.0576171875,
      "loss": 0.9128,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12403154373168945,
      "rewards/margins": -0.2896636128425598,
      "rewards/rejected": 0.16563208401203156,
      "step": 700
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.128518903328078e-05,
      "logits/chosen": -2.157421350479126,
      "logits/rejected": -2.1012282371520996,
      "logps/chosen": -165.8155517578125,
      "logps/rejected": -168.45291137695312,
      "loss": 0.9046,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.408125638961792,
      "rewards/margins": -0.2868077754974365,
      "rewards/rejected": -0.12131791561841965,
      "step": 701
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.125798693454396e-05,
      "logits/chosen": -2.077611207962036,
      "logits/rejected": -2.026050567626953,
      "logps/chosen": -142.8761749267578,
      "logps/rejected": -118.22064208984375,
      "loss": 0.7815,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.11512994021177292,
      "rewards/margins": -0.07021620869636536,
      "rewards/rejected": -0.044913746416568756,
      "step": 702
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.123075144050772e-05,
      "logits/chosen": -1.8848479986190796,
      "logits/rejected": -1.810481309890747,
      "logps/chosen": -165.26451110839844,
      "logps/rejected": -153.42242431640625,
      "loss": 0.7349,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6419331431388855,
      "rewards/margins": -0.010387040674686432,
      "rewards/rejected": -0.6315460801124573,
      "step": 703
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.120348260711611e-05,
      "logits/chosen": -2.085160255432129,
      "logits/rejected": -2.0904765129089355,
      "logps/chosen": -236.3929443359375,
      "logps/rejected": -242.44451904296875,
      "loss": 0.7578,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.37019020318984985,
      "rewards/margins": -0.07077489048242569,
      "rewards/rejected": -0.2994152903556824,
      "step": 704
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.117618049038165e-05,
      "logits/chosen": -2.1439950466156006,
      "logits/rejected": -2.22857666015625,
      "logps/chosen": -144.78225708007812,
      "logps/rejected": -181.41627502441406,
      "loss": 0.7095,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.03200235217809677,
      "rewards/margins": 0.057953305542469025,
      "rewards/rejected": -0.02595096081495285,
      "step": 705
    },
    {
      "epoch": 0.92,
      "learning_rate": 4.1148845146385214e-05,
      "logits/chosen": -2.0723843574523926,
      "logits/rejected": -2.1756937503814697,
      "logps/chosen": -138.92593383789062,
      "logps/rejected": -126.64840698242188,
      "loss": 0.8782,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.6072953939437866,
      "rewards/margins": -0.19861473143100739,
      "rewards/rejected": -0.40868061780929565,
      "step": 706
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.112147663127596e-05,
      "logits/chosen": -2.0885486602783203,
      "logits/rejected": -2.033440113067627,
      "logps/chosen": -152.20188903808594,
      "logps/rejected": -152.0369415283203,
      "loss": 0.6951,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.3034742772579193,
      "rewards/margins": 0.04874863103032112,
      "rewards/rejected": -0.35222291946411133,
      "step": 707
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.109407500127116e-05,
      "logits/chosen": -1.9431442022323608,
      "logits/rejected": -1.9316291809082031,
      "logps/chosen": -170.7488555908203,
      "logps/rejected": -174.8116455078125,
      "loss": 0.6792,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.2055988609790802,
      "rewards/margins": 0.07953611761331558,
      "rewards/rejected": 0.1260627806186676,
      "step": 708
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.106664031265611e-05,
      "logits/chosen": -2.0817761421203613,
      "logits/rejected": -2.1072683334350586,
      "logps/chosen": -163.31427001953125,
      "logps/rejected": -174.4534912109375,
      "loss": 0.5775,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.2241707593202591,
      "rewards/margins": 0.33371949195861816,
      "rewards/rejected": -0.10954873263835907,
      "step": 709
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.103917262178402e-05,
      "logits/chosen": -2.1241044998168945,
      "logits/rejected": -2.0995607376098633,
      "logps/chosen": -144.4809112548828,
      "logps/rejected": -157.41078186035156,
      "loss": 0.7782,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.273907333612442,
      "rewards/margins": -0.07136461138725281,
      "rewards/rejected": -0.20254270732402802,
      "step": 710
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.1011671985075865e-05,
      "logits/chosen": -1.778105616569519,
      "logits/rejected": -1.807044267654419,
      "logps/chosen": -171.52398681640625,
      "logps/rejected": -175.2049102783203,
      "loss": 0.9113,
      "rewards/accuracies": 0.1875,
      "rewards/chosen": -0.46856945753097534,
      "rewards/margins": -0.3406550884246826,
      "rewards/rejected": -0.12791432440280914,
      "step": 711
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.098413845902033e-05,
      "logits/chosen": -1.969050645828247,
      "logits/rejected": -1.9264585971832275,
      "logps/chosen": -175.43978881835938,
      "logps/rejected": -158.25021362304688,
      "loss": 0.8706,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.41940295696258545,
      "rewards/margins": -0.2438211590051651,
      "rewards/rejected": -0.17558182775974274,
      "step": 712
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.095657210017364e-05,
      "logits/chosen": -2.143251895904541,
      "logits/rejected": -2.111879587173462,
      "logps/chosen": -162.4940185546875,
      "logps/rejected": -155.96981811523438,
      "loss": 0.6832,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.04569421708583832,
      "rewards/margins": 0.11248306930065155,
      "rewards/rejected": -0.15817728638648987,
      "step": 713
    },
    {
      "epoch": 0.93,
      "learning_rate": 4.092897296515944e-05,
      "logits/chosen": -2.0187065601348877,
      "logits/rejected": -1.9244728088378906,
      "logps/chosen": -164.12301635742188,
      "logps/rejected": -184.9077606201172,
      "loss": 0.6384,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.13283205032348633,
      "rewards/margins": 0.15339265763759613,
      "rewards/rejected": -0.28622472286224365,
      "step": 714
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.090134111066874e-05,
      "logits/chosen": -1.8528801202774048,
      "logits/rejected": -1.8440043926239014,
      "logps/chosen": -179.0013427734375,
      "logps/rejected": -188.64830017089844,
      "loss": 0.7665,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.24608290195465088,
      "rewards/margins": -0.08188387751579285,
      "rewards/rejected": -0.16419902443885803,
      "step": 715
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.0873676593459725e-05,
      "logits/chosen": -1.9546207189559937,
      "logits/rejected": -1.875700831413269,
      "logps/chosen": -135.4862060546875,
      "logps/rejected": -141.05101013183594,
      "loss": 0.6278,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.23251035809516907,
      "rewards/margins": 0.21048937737941742,
      "rewards/rejected": -0.4429997205734253,
      "step": 716
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.08459794703577e-05,
      "logits/chosen": -2.0966317653656006,
      "logits/rejected": -2.0341391563415527,
      "logps/chosen": -151.62359619140625,
      "logps/rejected": -155.7061767578125,
      "loss": 0.6931,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1692691296339035,
      "rewards/margins": 0.05348172038793564,
      "rewards/rejected": -0.22275087237358093,
      "step": 717
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.081824979825492e-05,
      "logits/chosen": -2.116243362426758,
      "logits/rejected": -2.13222074508667,
      "logps/chosen": -147.01651000976562,
      "logps/rejected": -149.95855712890625,
      "loss": 0.7723,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2274257391691208,
      "rewards/margins": -0.09935702383518219,
      "rewards/rejected": -0.128068745136261,
      "step": 718
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.07904876341105e-05,
      "logits/chosen": -2.0546305179595947,
      "logits/rejected": -2.1076295375823975,
      "logps/chosen": -152.6702117919922,
      "logps/rejected": -143.9462127685547,
      "loss": 0.7773,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.06750668585300446,
      "rewards/margins": -0.091024249792099,
      "rewards/rejected": 0.023517563939094543,
      "step": 719
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.076269303495033e-05,
      "logits/chosen": -2.0860443115234375,
      "logits/rejected": -2.146270751953125,
      "logps/chosen": -143.63839721679688,
      "logps/rejected": -160.30484008789062,
      "loss": 0.6651,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.12404690682888031,
      "rewards/margins": 0.13515986502170563,
      "rewards/rejected": -0.2592068016529083,
      "step": 720
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.073486605786689e-05,
      "logits/chosen": -2.288916826248169,
      "logits/rejected": -2.24888277053833,
      "logps/chosen": -195.5283203125,
      "logps/rejected": -183.82054138183594,
      "loss": 0.712,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.2654836177825928,
      "rewards/margins": -0.004374176263809204,
      "rewards/rejected": -0.26110947132110596,
      "step": 721
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.0707006760019175e-05,
      "logits/chosen": -2.06491756439209,
      "logits/rejected": -2.0922203063964844,
      "logps/chosen": -151.57093811035156,
      "logps/rejected": -156.29843139648438,
      "loss": 0.7023,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.3546985387802124,
      "rewards/margins": 0.029046528041362762,
      "rewards/rejected": -0.38374507427215576,
      "step": 722
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.067911519863257e-05,
      "logits/chosen": -2.074341058731079,
      "logits/rejected": -2.069526433944702,
      "logps/chosen": -144.95079040527344,
      "logps/rejected": -143.27981567382812,
      "loss": 0.9193,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.13518384099006653,
      "rewards/margins": -0.2904050052165985,
      "rewards/rejected": 0.15522116422653198,
      "step": 723
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.065119143099874e-05,
      "logits/chosen": -2.0665669441223145,
      "logits/rejected": -2.0500869750976562,
      "logps/chosen": -129.72299194335938,
      "logps/rejected": -123.09598541259766,
      "loss": 0.6954,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1962793618440628,
      "rewards/margins": 0.07191440463066101,
      "rewards/rejected": -0.268193781375885,
      "step": 724
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.062323551447549e-05,
      "logits/chosen": -1.8880443572998047,
      "logits/rejected": -1.9188331365585327,
      "logps/chosen": -197.51145935058594,
      "logps/rejected": -205.3161163330078,
      "loss": 0.7255,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.244761124253273,
      "rewards/margins": 0.003908276557922363,
      "rewards/rejected": -0.24866940081119537,
      "step": 725
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.059524750648668e-05,
      "logits/chosen": -2.0474305152893066,
      "logits/rejected": -2.0502195358276367,
      "logps/chosen": -146.49232482910156,
      "logps/rejected": -169.22877502441406,
      "loss": 0.8212,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.12031140178442001,
      "rewards/margins": -0.1856772005558014,
      "rewards/rejected": 0.06536579132080078,
      "step": 726
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.056722746452207e-05,
      "logits/chosen": -1.802725076675415,
      "logits/rejected": -1.8228119611740112,
      "logps/chosen": -155.632080078125,
      "logps/rejected": -176.6995849609375,
      "loss": 0.9494,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.5103746652603149,
      "rewards/margins": -0.398013174533844,
      "rewards/rejected": -0.11236148327589035,
      "step": 727
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.053917544613723e-05,
      "logits/chosen": -2.0305285453796387,
      "logits/rejected": -1.9622552394866943,
      "logps/chosen": -155.14163208007812,
      "logps/rejected": -141.12828063964844,
      "loss": 0.7736,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.26194462180137634,
      "rewards/margins": -0.11262011528015137,
      "rewards/rejected": -0.1493244767189026,
      "step": 728
    },
    {
      "epoch": 0.95,
      "learning_rate": 4.051109150895343e-05,
      "logits/chosen": -2.125661611557007,
      "logits/rejected": -2.060500144958496,
      "logps/chosen": -136.96627807617188,
      "logps/rejected": -152.24871826171875,
      "loss": 0.6332,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.14833050966262817,
      "rewards/margins": 0.16227081418037415,
      "rewards/rejected": -0.3106013536453247,
      "step": 729
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.0482975710657455e-05,
      "logits/chosen": -2.14182710647583,
      "logits/rejected": -2.084892511367798,
      "logps/chosen": -175.79473876953125,
      "logps/rejected": -168.12074279785156,
      "loss": 0.8449,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.2244684100151062,
      "rewards/margins": -0.2297031730413437,
      "rewards/rejected": 0.0052347928285598755,
      "step": 730
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.045482810900159e-05,
      "logits/chosen": -2.0067529678344727,
      "logits/rejected": -2.0317680835723877,
      "logps/chosen": -168.8496856689453,
      "logps/rejected": -163.8263397216797,
      "loss": 0.6518,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.19057075679302216,
      "rewards/margins": 0.1778174489736557,
      "rewards/rejected": -0.36838820576667786,
      "step": 731
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.042664876180341e-05,
      "logits/chosen": -2.1688036918640137,
      "logits/rejected": -2.1718666553497314,
      "logps/chosen": -170.41114807128906,
      "logps/rejected": -169.48265075683594,
      "loss": 0.8418,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.15957602858543396,
      "rewards/margins": -0.18063148856163025,
      "rewards/rejected": 0.021055400371551514,
      "step": 732
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.0398437726945716e-05,
      "logits/chosen": -1.8246833086013794,
      "logits/rejected": -1.7999447584152222,
      "logps/chosen": -142.14801025390625,
      "logps/rejected": -152.80712890625,
      "loss": 0.8179,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.20929935574531555,
      "rewards/margins": -0.17062106728553772,
      "rewards/rejected": -0.03867826238274574,
      "step": 733
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.037019506237638e-05,
      "logits/chosen": -1.6244423389434814,
      "logits/rejected": -1.6328479051589966,
      "logps/chosen": -172.3773956298828,
      "logps/rejected": -212.7267303466797,
      "loss": 0.5395,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.04836279898881912,
      "rewards/margins": 0.5529451370239258,
      "rewards/rejected": -0.6013079881668091,
      "step": 734
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.034192082610828e-05,
      "logits/chosen": -2.0388615131378174,
      "logits/rejected": -2.085139513015747,
      "logps/chosen": -141.1918182373047,
      "logps/rejected": -139.84365844726562,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08627957850694656,
      "rewards/margins": 0.07983030378818512,
      "rewards/rejected": 0.006449267268180847,
      "step": 735
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.031361507621911e-05,
      "logits/chosen": -2.163036584854126,
      "logits/rejected": -2.172485589981079,
      "logps/chosen": -149.1195068359375,
      "logps/rejected": -136.47434997558594,
      "loss": 0.7033,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.05051477998495102,
      "rewards/margins": 0.08880610764026642,
      "rewards/rejected": -0.0382913202047348,
      "step": 736
    },
    {
      "epoch": 0.96,
      "learning_rate": 4.02852778708513e-05,
      "logits/chosen": -1.6721755266189575,
      "logits/rejected": -1.6523504257202148,
      "logps/chosen": -180.20445251464844,
      "logps/rejected": -192.36607360839844,
      "loss": 0.7664,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.17255781590938568,
      "rewards/margins": -0.05452100187540054,
      "rewards/rejected": -0.11803679168224335,
      "step": 737
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.0256909268211914e-05,
      "logits/chosen": -2.0831446647644043,
      "logits/rejected": -2.0978503227233887,
      "logps/chosen": -121.0216064453125,
      "logps/rejected": -125.146484375,
      "loss": 0.7649,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.14153949916362762,
      "rewards/margins": -0.12446232885122299,
      "rewards/rejected": -0.017077183350920677,
      "step": 738
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.0228509326572496e-05,
      "logits/chosen": -1.6901588439941406,
      "logits/rejected": -1.721649408340454,
      "logps/chosen": -134.27215576171875,
      "logps/rejected": -136.54830932617188,
      "loss": 0.6938,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.012902306392788887,
      "rewards/margins": 0.09037284553050995,
      "rewards/rejected": -0.07747054100036621,
      "step": 739
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.0200078104268944e-05,
      "logits/chosen": -2.149341344833374,
      "logits/rejected": -2.117537021636963,
      "logps/chosen": -161.55764770507812,
      "logps/rejected": -142.29367065429688,
      "loss": 0.8426,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.18234218657016754,
      "rewards/margins": -0.15283283591270447,
      "rewards/rejected": -0.02950935624539852,
      "step": 740
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.017161565970144e-05,
      "logits/chosen": -1.7856075763702393,
      "logits/rejected": -1.7834945917129517,
      "logps/chosen": -158.6090545654297,
      "logps/rejected": -158.350341796875,
      "loss": 0.7749,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.17489013075828552,
      "rewards/margins": -0.10388021171092987,
      "rewards/rejected": -0.07100993394851685,
      "step": 741
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.014312205133428e-05,
      "logits/chosen": -2.026144027709961,
      "logits/rejected": -2.0260369777679443,
      "logps/chosen": -169.1302490234375,
      "logps/rejected": -125.34761047363281,
      "loss": 0.778,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": 0.043028444051742554,
      "rewards/margins": -0.08100062608718872,
      "rewards/rejected": 0.12402907758951187,
      "step": 742
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.011459733769579e-05,
      "logits/chosen": -1.9844965934753418,
      "logits/rejected": -2.031277656555176,
      "logps/chosen": -139.288818359375,
      "logps/rejected": -152.935791015625,
      "loss": 0.8053,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.28731998801231384,
      "rewards/margins": -0.16818708181381226,
      "rewards/rejected": -0.11913290619850159,
      "step": 743
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.0086041577378166e-05,
      "logits/chosen": -1.9718449115753174,
      "logits/rejected": -2.001786708831787,
      "logps/chosen": -113.56793212890625,
      "logps/rejected": -115.19658660888672,
      "loss": 0.6771,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10746096074581146,
      "rewards/margins": 0.0731014758348465,
      "rewards/rejected": -0.18056243658065796,
      "step": 744
    },
    {
      "epoch": 0.97,
      "learning_rate": 4.005745482903739e-05,
      "logits/chosen": -1.9297832250595093,
      "logits/rejected": -1.8905633687973022,
      "logps/chosen": -189.31784057617188,
      "logps/rejected": -230.98753356933594,
      "loss": 0.6559,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.15024004876613617,
      "rewards/margins": 0.3164423704147339,
      "rewards/rejected": -0.466682493686676,
      "step": 745
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.002883715139309e-05,
      "logits/chosen": -1.7887859344482422,
      "logits/rejected": -1.82424795627594,
      "logps/chosen": -208.4186553955078,
      "logps/rejected": -213.05670166015625,
      "loss": 0.6253,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.026865793392062187,
      "rewards/margins": 0.1876009702682495,
      "rewards/rejected": -0.16073518991470337,
      "step": 746
    },
    {
      "epoch": 0.98,
      "learning_rate": 4.000018860322845e-05,
      "logits/chosen": -2.2344820499420166,
      "logits/rejected": -2.233741283416748,
      "logps/chosen": -183.4325408935547,
      "logps/rejected": -181.87393188476562,
      "loss": 0.5643,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.1357138305902481,
      "rewards/margins": 0.355926513671875,
      "rewards/rejected": -0.22021271288394928,
      "step": 747
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.9971509243390025e-05,
      "logits/chosen": -2.194760799407959,
      "logits/rejected": -2.2292375564575195,
      "logps/chosen": -156.87254333496094,
      "logps/rejected": -168.62025451660156,
      "loss": 0.7486,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.046510808169841766,
      "rewards/margins": 0.025812923908233643,
      "rewards/rejected": -0.07232370227575302,
      "step": 748
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.99427991307877e-05,
      "logits/chosen": -1.8178576231002808,
      "logits/rejected": -1.8639076948165894,
      "logps/chosen": -148.9747314453125,
      "logps/rejected": -161.0830078125,
      "loss": 0.6006,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.05070438235998154,
      "rewards/margins": 0.26105034351348877,
      "rewards/rejected": -0.21034595370292664,
      "step": 749
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.9914058324394486e-05,
      "logits/chosen": -2.137637138366699,
      "logits/rejected": -2.1838955879211426,
      "logps/chosen": -171.7498016357422,
      "logps/rejected": -182.58238220214844,
      "loss": 0.628,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.053341031074523926,
      "rewards/margins": 0.19571509957313538,
      "rewards/rejected": -0.14237406849861145,
      "step": 750
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.9885286883246476e-05,
      "logits/chosen": -1.965986728668213,
      "logits/rejected": -2.001103162765503,
      "logps/chosen": -151.49935913085938,
      "logps/rejected": -180.28414916992188,
      "loss": 0.7626,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.19454632699489594,
      "rewards/margins": -0.05682283639907837,
      "rewards/rejected": -0.13772347569465637,
      "step": 751
    },
    {
      "epoch": 0.98,
      "learning_rate": 3.985648486644267e-05,
      "logits/chosen": -2.0965442657470703,
      "logits/rejected": -2.115654706954956,
      "logps/chosen": -153.47987365722656,
      "logps/rejected": -161.52932739257812,
      "loss": 0.8653,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.03188393637537956,
      "rewards/margins": -0.20181989669799805,
      "rewards/rejected": 0.169935941696167,
      "step": 752
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.982765233314489e-05,
      "logits/chosen": -1.8265806436538696,
      "logits/rejected": -1.8221160173416138,
      "logps/chosen": -201.52655029296875,
      "logps/rejected": -200.19342041015625,
      "loss": 0.7038,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.10736799240112305,
      "rewards/margins": 0.025161506608128548,
      "rewards/rejected": -0.13252949714660645,
      "step": 753
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.979878934257762e-05,
      "logits/chosen": -1.97774076461792,
      "logits/rejected": -1.9710618257522583,
      "logps/chosen": -161.500732421875,
      "logps/rejected": -155.46812438964844,
      "loss": 0.7226,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.40990006923675537,
      "rewards/margins": 0.09453297406435013,
      "rewards/rejected": -0.5044330954551697,
      "step": 754
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.976989595402793e-05,
      "logits/chosen": -1.9539647102355957,
      "logits/rejected": -1.8984624147415161,
      "logps/chosen": -156.4872283935547,
      "logps/rejected": -157.50094604492188,
      "loss": 1.0053,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.43122974038124084,
      "rewards/margins": -0.45546600222587585,
      "rewards/rejected": 0.024236250668764114,
      "step": 755
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.974097222684532e-05,
      "logits/chosen": -2.0011062622070312,
      "logits/rejected": -2.0779013633728027,
      "logps/chosen": -142.67002868652344,
      "logps/rejected": -146.49705505371094,
      "loss": 0.7767,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.021236799657344818,
      "rewards/margins": -0.09352488815784454,
      "rewards/rejected": 0.07228809595108032,
      "step": 756
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.9712018220441596e-05,
      "logits/chosen": -2.1120223999023438,
      "logits/rejected": -2.0883326530456543,
      "logps/chosen": -150.02957153320312,
      "logps/rejected": -154.8041534423828,
      "loss": 0.8471,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.38133108615875244,
      "rewards/margins": -0.2452768087387085,
      "rewards/rejected": -0.13605426251888275,
      "step": 757
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.9683033994290767e-05,
      "logits/chosen": -1.9895920753479004,
      "logits/rejected": -2.0780937671661377,
      "logps/chosen": -122.83441925048828,
      "logps/rejected": -135.78131103515625,
      "loss": 0.7101,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.042702965438365936,
      "rewards/margins": 0.10642305016517639,
      "rewards/rejected": -0.14912599325180054,
      "step": 758
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.965401960792894e-05,
      "logits/chosen": -1.9442851543426514,
      "logits/rejected": -1.9203828573226929,
      "logps/chosen": -163.14080810546875,
      "logps/rejected": -158.33706665039062,
      "loss": 0.7844,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.21191173791885376,
      "rewards/margins": -0.12215844541788101,
      "rewards/rejected": -0.08975328505039215,
      "step": 759
    },
    {
      "epoch": 0.99,
      "learning_rate": 3.962497512095412e-05,
      "logits/chosen": -2.0754759311676025,
      "logits/rejected": -2.033245801925659,
      "logps/chosen": -209.28817749023438,
      "logps/rejected": -203.22897338867188,
      "loss": 0.8654,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.20608650147914886,
      "rewards/margins": -0.2308453768491745,
      "rewards/rejected": 0.02475885860621929,
      "step": 760
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.95959005930262e-05,
      "logits/chosen": -1.9952623844146729,
      "logits/rejected": -1.9730302095413208,
      "logps/chosen": -192.40951538085938,
      "logps/rejected": -196.0675506591797,
      "loss": 0.7612,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.10101304203271866,
      "rewards/margins": -0.0519864559173584,
      "rewards/rejected": -0.04902658611536026,
      "step": 761
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.9566796083866756e-05,
      "logits/chosen": -1.8687723875045776,
      "logits/rejected": -1.8516573905944824,
      "logps/chosen": -163.812744140625,
      "logps/rejected": -167.4736328125,
      "loss": 0.6654,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08760890364646912,
      "rewards/margins": 0.10946011543273926,
      "rewards/rejected": -0.02185121551156044,
      "step": 762
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.953766165325892e-05,
      "logits/chosen": -1.9016671180725098,
      "logits/rejected": -1.8528916835784912,
      "logps/chosen": -122.68069458007812,
      "logps/rejected": -166.39230346679688,
      "loss": 0.8049,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.1457635462284088,
      "rewards/margins": -0.134870246052742,
      "rewards/rejected": -0.010893300175666809,
      "step": 763
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.9508497361047334e-05,
      "logits/chosen": -2.126763105392456,
      "logits/rejected": -2.0734221935272217,
      "logps/chosen": -153.5342254638672,
      "logps/rejected": -124.99877166748047,
      "loss": 0.7467,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09060416370630264,
      "rewards/margins": -0.037557389587163925,
      "rewards/rejected": -0.05304676294326782,
      "step": 764
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.9479303267137944e-05,
      "logits/chosen": -2.080320358276367,
      "logits/rejected": -2.0969486236572266,
      "logps/chosen": -181.7772216796875,
      "logps/rejected": -209.83665466308594,
      "loss": 0.1747,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.5646958351135254,
      "rewards/margins": 3.972540855407715,
      "rewards/rejected": -2.4078450202941895,
      "step": 765
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.9450079431497936e-05,
      "logits/chosen": -2.0887792110443115,
      "logits/rejected": -2.027618169784546,
      "logps/chosen": -161.34857177734375,
      "logps/rejected": -186.8290252685547,
      "loss": 0.0359,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.125178098678589,
      "rewards/margins": 5.42678165435791,
      "rewards/rejected": -3.301603317260742,
      "step": 766
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.9420825914155554e-05,
      "logits/chosen": -2.104077100753784,
      "logits/rejected": -2.1051535606384277,
      "logps/chosen": -139.18788146972656,
      "logps/rejected": -189.3860321044922,
      "loss": 0.124,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.9335505962371826,
      "rewards/margins": 4.618389129638672,
      "rewards/rejected": -2.6848387718200684,
      "step": 767
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.939154277520006e-05,
      "logits/chosen": -2.1465070247650146,
      "logits/rejected": -2.0535991191864014,
      "logps/chosen": -140.7772979736328,
      "logps/rejected": -178.20533752441406,
      "loss": 0.0353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.0950372219085693,
      "rewards/margins": 4.902841091156006,
      "rewards/rejected": -2.8078043460845947,
      "step": 768
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.9362230074781506e-05,
      "logits/chosen": -2.032116413116455,
      "logits/rejected": -2.0662102699279785,
      "logps/chosen": -163.39682006835938,
      "logps/rejected": -208.9725341796875,
      "loss": 0.1048,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.8455992937088013,
      "rewards/margins": 4.063425064086914,
      "rewards/rejected": -2.2178261280059814,
      "step": 769
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.9332887873110695e-05,
      "logits/chosen": -1.9861818552017212,
      "logits/rejected": -2.0381555557250977,
      "logps/chosen": -171.25347900390625,
      "logps/rejected": -219.2564239501953,
      "loss": 0.0385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 2.1809895038604736,
      "rewards/margins": 6.388942718505859,
      "rewards/rejected": -4.207952976226807,
      "step": 770
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.9303516230459035e-05,
      "logits/chosen": -2.1124377250671387,
      "logits/rejected": -2.0706801414489746,
      "logps/chosen": -129.31185913085938,
      "logps/rejected": -167.8997039794922,
      "loss": 0.137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.6835626363754272,
      "rewards/margins": 4.556663990020752,
      "rewards/rejected": -2.8731014728546143,
      "step": 771
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.92741152071584e-05,
      "logits/chosen": -1.8337211608886719,
      "logits/rejected": -1.8092454671859741,
      "logps/chosen": -136.34719848632812,
      "logps/rejected": -187.0718994140625,
      "loss": 0.1101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5944790840148926,
      "rewards/margins": 4.034244537353516,
      "rewards/rejected": -2.439765214920044,
      "step": 772
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.924468486360101e-05,
      "logits/chosen": -1.8283705711364746,
      "logits/rejected": -1.7894047498703003,
      "logps/chosen": -245.77256774902344,
      "logps/rejected": -281.06781005859375,
      "loss": 0.1771,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.18741774559021,
      "rewards/margins": 3.2372922897338867,
      "rewards/rejected": -2.0498743057250977,
      "step": 773
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.921522526023931e-05,
      "logits/chosen": -1.9555280208587646,
      "logits/rejected": -2.059631586074829,
      "logps/chosen": -144.19586181640625,
      "logps/rejected": -203.6154327392578,
      "loss": 0.0749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2664604187011719,
      "rewards/margins": 3.9877490997314453,
      "rewards/rejected": -2.7212884426116943,
      "step": 774
    },
    {
      "epoch": 1.01,
      "learning_rate": 3.918573645758586e-05,
      "logits/chosen": -2.0166711807250977,
      "logits/rejected": -2.075800657272339,
      "logps/chosen": -155.22332763671875,
      "logps/rejected": -199.52328491210938,
      "loss": 0.1121,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.6841697692871094,
      "rewards/margins": 4.346258640289307,
      "rewards/rejected": -2.6620891094207764,
      "step": 775
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.915621851621318e-05,
      "logits/chosen": -1.8551626205444336,
      "logits/rejected": -1.8971370458602905,
      "logps/chosen": -144.870849609375,
      "logps/rejected": -215.3363037109375,
      "loss": 0.0826,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.5322911739349365,
      "rewards/margins": 4.17916202545166,
      "rewards/rejected": -2.6468706130981445,
      "step": 776
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.9126671496753666e-05,
      "logits/chosen": -1.8935290575027466,
      "logits/rejected": -1.8119558095932007,
      "logps/chosen": -175.02459716796875,
      "logps/rejected": -182.73126220703125,
      "loss": 0.1091,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.9930130243301392,
      "rewards/margins": 4.418476581573486,
      "rewards/rejected": -2.4254636764526367,
      "step": 777
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.909709545989942e-05,
      "logits/chosen": -1.8846909999847412,
      "logits/rejected": -1.8772252798080444,
      "logps/chosen": -123.37010192871094,
      "logps/rejected": -165.76968383789062,
      "loss": 0.0891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.5085937976837158,
      "rewards/margins": 4.403595924377441,
      "rewards/rejected": -2.8950023651123047,
      "step": 778
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.9067490466402156e-05,
      "logits/chosen": -1.924761414527893,
      "logits/rejected": -1.9515211582183838,
      "logps/chosen": -119.23345947265625,
      "logps/rejected": -164.5015869140625,
      "loss": 0.178,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.1522871255874634,
      "rewards/margins": 3.376887083053589,
      "rewards/rejected": -2.224600076675415,
      "step": 779
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.903785657707307e-05,
      "logits/chosen": -1.7551360130310059,
      "logits/rejected": -1.8034405708312988,
      "logps/chosen": -132.70225524902344,
      "logps/rejected": -188.23854064941406,
      "loss": 0.1239,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.4596197605133057,
      "rewards/margins": 5.111968994140625,
      "rewards/rejected": -3.6523497104644775,
      "step": 780
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.9008193852782733e-05,
      "logits/chosen": -1.9345866441726685,
      "logits/rejected": -2.0259900093078613,
      "logps/chosen": -147.717529296875,
      "logps/rejected": -198.76608276367188,
      "loss": 0.1756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 1.160006046295166,
      "rewards/margins": 3.809837579727173,
      "rewards/rejected": -2.649831533432007,
      "step": 781
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.897850235446089e-05,
      "logits/chosen": -1.8931665420532227,
      "logits/rejected": -1.8667347431182861,
      "logps/chosen": -144.32681274414062,
      "logps/rejected": -176.2711181640625,
      "loss": 0.0913,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.2157329320907593,
      "rewards/margins": 4.00248908996582,
      "rewards/rejected": -2.7867562770843506,
      "step": 782
    },
    {
      "epoch": 1.02,
      "learning_rate": 3.894878214309645e-05,
      "logits/chosen": -1.8950620889663696,
      "logits/rejected": -1.7776212692260742,
      "logps/chosen": -153.48272705078125,
      "logps/rejected": -185.09901428222656,
      "loss": 0.0948,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.7234764099121094,
      "rewards/margins": 4.474335193634033,
      "rewards/rejected": -2.7508585453033447,
      "step": 783
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.8919033279737274e-05,
      "logits/chosen": -1.9167214632034302,
      "logits/rejected": -1.955217719078064,
      "logps/chosen": -149.79092407226562,
      "logps/rejected": -209.3562469482422,
      "loss": 0.1909,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.6991943717002869,
      "rewards/margins": 3.995697021484375,
      "rewards/rejected": -3.2965028285980225,
      "step": 784
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.888925582549006e-05,
      "logits/chosen": -1.7585481405258179,
      "logits/rejected": -1.7582013607025146,
      "logps/chosen": -161.952392578125,
      "logps/rejected": -223.1368408203125,
      "loss": 0.0444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.2704546451568604,
      "rewards/margins": 5.138402462005615,
      "rewards/rejected": -3.867948055267334,
      "step": 785
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.885944984152027e-05,
      "logits/chosen": -1.7177354097366333,
      "logits/rejected": -1.7072687149047852,
      "logps/chosen": -141.16017150878906,
      "logps/rejected": -197.3968963623047,
      "loss": 0.0631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3352869749069214,
      "rewards/margins": 4.8079514503479,
      "rewards/rejected": -3.4726650714874268,
      "step": 786
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.882961538905194e-05,
      "logits/chosen": -1.9421478509902954,
      "logits/rejected": -1.9236348867416382,
      "logps/chosen": -156.8341827392578,
      "logps/rejected": -201.7722625732422,
      "loss": 0.0734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 1.3398832082748413,
      "rewards/margins": 4.717867851257324,
      "rewards/rejected": -3.3779852390289307,
      "step": 787
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.879975252936761e-05,
      "logits/chosen": -1.918500542640686,
      "logits/rejected": -1.9371358156204224,
      "logps/chosen": -200.98944091796875,
      "logps/rejected": -276.0814514160156,
      "loss": 0.0714,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0912672281265259,
      "rewards/margins": 5.926272392272949,
      "rewards/rejected": -4.835005283355713,
      "step": 788
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.876986132380814e-05,
      "logits/chosen": -1.894209623336792,
      "logits/rejected": -1.9814263582229614,
      "logps/chosen": -127.55168151855469,
      "logps/rejected": -186.98684692382812,
      "loss": 0.0765,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0201938152313232,
      "rewards/margins": 4.509597301483154,
      "rewards/rejected": -3.489403247833252,
      "step": 789
    },
    {
      "epoch": 1.03,
      "learning_rate": 3.8739941833772643e-05,
      "logits/chosen": -1.743139386177063,
      "logits/rejected": -1.7789626121520996,
      "logps/chosen": -135.5447235107422,
      "logps/rejected": -201.73886108398438,
      "loss": 0.1591,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7540079951286316,
      "rewards/margins": 5.139564514160156,
      "rewards/rejected": -4.385556697845459,
      "step": 790
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.870999412071829e-05,
      "logits/chosen": -1.9243521690368652,
      "logits/rejected": -1.907222867012024,
      "logps/chosen": -132.49636840820312,
      "logps/rejected": -192.06646728515625,
      "loss": 0.1116,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.1499325037002563,
      "rewards/margins": 5.0751800537109375,
      "rewards/rejected": -3.92524790763855,
      "step": 791
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.8680018246160295e-05,
      "logits/chosen": -1.6514546871185303,
      "logits/rejected": -1.6387147903442383,
      "logps/chosen": -132.7026824951172,
      "logps/rejected": -164.41177368164062,
      "loss": 0.112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4314553737640381,
      "rewards/margins": 3.427611827850342,
      "rewards/rejected": -2.9961564540863037,
      "step": 792
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.865001427167164e-05,
      "logits/chosen": -1.8426374197006226,
      "logits/rejected": -1.8846511840820312,
      "logps/chosen": -136.38165283203125,
      "logps/rejected": -199.7681121826172,
      "loss": 0.0956,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.0450799465179443,
      "rewards/margins": 5.463000297546387,
      "rewards/rejected": -4.4179205894470215,
      "step": 793
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.861998225888307e-05,
      "logits/chosen": -1.8042436838150024,
      "logits/rejected": -1.8148198127746582,
      "logps/chosen": -142.37742614746094,
      "logps/rejected": -197.7374267578125,
      "loss": 0.0785,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.9900847673416138,
      "rewards/margins": 5.896683692932129,
      "rewards/rejected": -4.906599044799805,
      "step": 794
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.8589922269482924e-05,
      "logits/chosen": -1.9162298440933228,
      "logits/rejected": -1.9350849390029907,
      "logps/chosen": -147.20562744140625,
      "logps/rejected": -187.5604705810547,
      "loss": 0.0646,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 1.3646960258483887,
      "rewards/margins": 4.906591415405273,
      "rewards/rejected": -3.5418953895568848,
      "step": 795
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.855983436521699e-05,
      "logits/chosen": -1.5755800008773804,
      "logits/rejected": -1.5610840320587158,
      "logps/chosen": -128.9073028564453,
      "logps/rejected": -180.30923461914062,
      "loss": 0.1132,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.7019076943397522,
      "rewards/margins": 4.735352039337158,
      "rewards/rejected": -4.033443927764893,
      "step": 796
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.8529718607888394e-05,
      "logits/chosen": -2.063124656677246,
      "logits/rejected": -2.062586545944214,
      "logps/chosen": -145.98892211914062,
      "logps/rejected": -188.89068603515625,
      "loss": 0.1308,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6006640791893005,
      "rewards/margins": 4.575399875640869,
      "rewards/rejected": -3.974735975265503,
      "step": 797
    },
    {
      "epoch": 1.04,
      "learning_rate": 3.8499575059357506e-05,
      "logits/chosen": -2.017670154571533,
      "logits/rejected": -1.987682819366455,
      "logps/chosen": -162.8350830078125,
      "logps/rejected": -233.1547393798828,
      "loss": 0.0839,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6563400030136108,
      "rewards/margins": 4.883859157562256,
      "rewards/rejected": -4.227519512176514,
      "step": 798
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.8469403781541745e-05,
      "logits/chosen": -1.8146461248397827,
      "logits/rejected": -1.7456778287887573,
      "logps/chosen": -230.2406005859375,
      "logps/rejected": -258.3357849121094,
      "loss": 0.129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0335506796836853,
      "rewards/margins": 4.429178714752197,
      "rewards/rejected": -4.395627975463867,
      "step": 799
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.843920483641551e-05,
      "logits/chosen": -1.9329719543457031,
      "logits/rejected": -1.9156296253204346,
      "logps/chosen": -136.6041717529297,
      "logps/rejected": -176.88087463378906,
      "loss": 0.0892,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.7322125434875488,
      "rewards/margins": 5.226524353027344,
      "rewards/rejected": -4.494311332702637,
      "step": 800
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.840897828601002e-05,
      "logits/chosen": -1.6023508310317993,
      "logits/rejected": -1.6873928308486938,
      "logps/chosen": -116.11083984375,
      "logps/rejected": -189.20831298828125,
      "loss": 0.0576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5114001035690308,
      "rewards/margins": 5.1929192543029785,
      "rewards/rejected": -4.6815185546875,
      "step": 801
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.83787241924132e-05,
      "logits/chosen": -1.798394799232483,
      "logits/rejected": -1.797690987586975,
      "logps/chosen": -149.56710815429688,
      "logps/rejected": -228.49143981933594,
      "loss": 0.0177,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.9114323854446411,
      "rewards/margins": 6.009431838989258,
      "rewards/rejected": -5.097999572753906,
      "step": 802
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.8348442617769564e-05,
      "logits/chosen": -1.9224333763122559,
      "logits/rejected": -1.9393596649169922,
      "logps/chosen": -152.3304443359375,
      "logps/rejected": -205.50289916992188,
      "loss": 0.0952,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3278241455554962,
      "rewards/margins": 5.112401485443115,
      "rewards/rejected": -4.784577369689941,
      "step": 803
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.831813362428005e-05,
      "logits/chosen": -1.937591552734375,
      "logits/rejected": -1.858033299446106,
      "logps/chosen": -138.99546813964844,
      "logps/rejected": -170.10922241210938,
      "loss": 0.1423,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.07943601906299591,
      "rewards/margins": 3.7533488273620605,
      "rewards/rejected": -3.673912763595581,
      "step": 804
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.8287797274201934e-05,
      "logits/chosen": -1.9712400436401367,
      "logits/rejected": -2.0466256141662598,
      "logps/chosen": -146.24937438964844,
      "logps/rejected": -204.24668884277344,
      "loss": 0.0868,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24946361780166626,
      "rewards/margins": 4.4833879470825195,
      "rewards/rejected": -4.23392391204834,
      "step": 805
    },
    {
      "epoch": 1.05,
      "learning_rate": 3.825743362984868e-05,
      "logits/chosen": -1.6642037630081177,
      "logits/rejected": -1.7018256187438965,
      "logps/chosen": -148.9640655517578,
      "logps/rejected": -202.5922088623047,
      "loss": 0.1762,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.22627446055412292,
      "rewards/margins": 5.173376560211182,
      "rewards/rejected": -4.9471025466918945,
      "step": 806
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.8227042753589824e-05,
      "logits/chosen": -1.74186372756958,
      "logits/rejected": -1.7566051483154297,
      "logps/chosen": -140.58749389648438,
      "logps/rejected": -201.23391723632812,
      "loss": 0.0873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41862282156944275,
      "rewards/margins": 5.055604457855225,
      "rewards/rejected": -4.636981010437012,
      "step": 807
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.819662470785082e-05,
      "logits/chosen": -1.9721858501434326,
      "logits/rejected": -1.9488880634307861,
      "logps/chosen": -197.06591796875,
      "logps/rejected": -247.65264892578125,
      "loss": 0.102,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2280472218990326,
      "rewards/margins": 6.295541763305664,
      "rewards/rejected": -6.067493915557861,
      "step": 808
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.816617955511296e-05,
      "logits/chosen": -1.9263876676559448,
      "logits/rejected": -2.024251937866211,
      "logps/chosen": -144.51605224609375,
      "logps/rejected": -229.19000244140625,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0787353515625,
      "rewards/margins": 6.103402137756348,
      "rewards/rejected": -6.1821370124816895,
      "step": 809
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.8135707357913176e-05,
      "logits/chosen": -1.9594213962554932,
      "logits/rejected": -1.9519600868225098,
      "logps/chosen": -163.80809020996094,
      "logps/rejected": -218.344970703125,
      "loss": 0.0908,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4812394976615906,
      "rewards/margins": 6.453742027282715,
      "rewards/rejected": -5.9725022315979,
      "step": 810
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.8105208178843984e-05,
      "logits/chosen": -1.9472453594207764,
      "logits/rejected": -1.9247050285339355,
      "logps/chosen": -176.8766326904297,
      "logps/rejected": -215.28683471679688,
      "loss": 0.0745,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.35657447576522827,
      "rewards/margins": 5.61616849899292,
      "rewards/rejected": -5.972743034362793,
      "step": 811
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.8074682080553335e-05,
      "logits/chosen": -2.037539005279541,
      "logits/rejected": -2.0407166481018066,
      "logps/chosen": -158.74447631835938,
      "logps/rejected": -200.7305145263672,
      "loss": 0.1276,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5493102073669434,
      "rewards/margins": 4.338137149810791,
      "rewards/rejected": -4.887447357177734,
      "step": 812
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.804412912574442e-05,
      "logits/chosen": -1.5923181772232056,
      "logits/rejected": -1.6689324378967285,
      "logps/chosen": -123.06846618652344,
      "logps/rejected": -194.64022827148438,
      "loss": 0.1177,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14290863275527954,
      "rewards/margins": 6.073092460632324,
      "rewards/rejected": -5.930182456970215,
      "step": 813
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.801354937717565e-05,
      "logits/chosen": -1.763596773147583,
      "logits/rejected": -1.761218547821045,
      "logps/chosen": -152.07403564453125,
      "logps/rejected": -195.23391723632812,
      "loss": 0.0384,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.022281453013420105,
      "rewards/margins": 5.0504584312438965,
      "rewards/rejected": -5.028176784515381,
      "step": 814
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.798294289766043e-05,
      "logits/chosen": -1.8372154235839844,
      "logits/rejected": -1.877131700515747,
      "logps/chosen": -153.13719177246094,
      "logps/rejected": -214.76300048828125,
      "loss": 0.164,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.30196332931518555,
      "rewards/margins": 5.951177597045898,
      "rewards/rejected": -6.253140449523926,
      "step": 815
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.795230975006712e-05,
      "logits/chosen": -2.02732253074646,
      "logits/rejected": -2.0462913513183594,
      "logps/chosen": -149.4512176513672,
      "logps/rejected": -206.4884490966797,
      "loss": 0.0554,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3465498387813568,
      "rewards/margins": 5.401089668273926,
      "rewards/rejected": -5.7476396560668945,
      "step": 816
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.792164999731881e-05,
      "logits/chosen": -1.9269187450408936,
      "logits/rejected": -1.9644137620925903,
      "logps/chosen": -130.3570556640625,
      "logps/rejected": -184.312255859375,
      "loss": 0.1769,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5219983458518982,
      "rewards/margins": 4.185467720031738,
      "rewards/rejected": -4.707466125488281,
      "step": 817
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.789096370239328e-05,
      "logits/chosen": -1.7042335271835327,
      "logits/rejected": -1.730513334274292,
      "logps/chosen": -195.3024139404297,
      "logps/rejected": -241.0108184814453,
      "loss": 0.0231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1243710145354271,
      "rewards/margins": 6.514937400817871,
      "rewards/rejected": -6.390565872192383,
      "step": 818
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.786025092832279e-05,
      "logits/chosen": -1.6639565229415894,
      "logits/rejected": -1.5779391527175903,
      "logps/chosen": -174.08763122558594,
      "logps/rejected": -248.2294464111328,
      "loss": 0.0325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5761074423789978,
      "rewards/margins": 7.08044958114624,
      "rewards/rejected": -7.656557083129883,
      "step": 819
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.782951173819403e-05,
      "logits/chosen": -1.858964204788208,
      "logits/rejected": -1.8241450786590576,
      "logps/chosen": -175.27001953125,
      "logps/rejected": -236.11073303222656,
      "loss": 0.0821,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.46175456047058105,
      "rewards/margins": 6.477173805236816,
      "rewards/rejected": -6.938928604125977,
      "step": 820
    },
    {
      "epoch": 1.07,
      "learning_rate": 3.7798746195147914e-05,
      "logits/chosen": -1.890723466873169,
      "logits/rejected": -1.931648850440979,
      "logps/chosen": -148.67684936523438,
      "logps/rejected": -197.44638061523438,
      "loss": 0.0738,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4770905375480652,
      "rewards/margins": 4.815718173980713,
      "rewards/rejected": -5.292808532714844,
      "step": 821
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.776795436237954e-05,
      "logits/chosen": -1.5346938371658325,
      "logits/rejected": -1.5795890092849731,
      "logps/chosen": -134.60568237304688,
      "logps/rejected": -227.31204223632812,
      "loss": 0.0891,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.7744713425636292,
      "rewards/margins": 8.283700942993164,
      "rewards/rejected": -7.50922966003418,
      "step": 822
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.773713630313793e-05,
      "logits/chosen": -1.794039011001587,
      "logits/rejected": -1.7841002941131592,
      "logps/chosen": -177.0500946044922,
      "logps/rejected": -220.89031982421875,
      "loss": 0.1019,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4332965612411499,
      "rewards/margins": 7.096560001373291,
      "rewards/rejected": -7.5298566818237305,
      "step": 823
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.7706292080726055e-05,
      "logits/chosen": -1.6856433153152466,
      "logits/rejected": -1.6186890602111816,
      "logps/chosen": -180.94700622558594,
      "logps/rejected": -231.9288330078125,
      "loss": 0.0917,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3657735586166382,
      "rewards/margins": 7.685838222503662,
      "rewards/rejected": -8.051612854003906,
      "step": 824
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.767542175850058e-05,
      "logits/chosen": -1.6293786764144897,
      "logits/rejected": -1.6187849044799805,
      "logps/chosen": -164.32154846191406,
      "logps/rejected": -270.12139892578125,
      "loss": 0.0215,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7973673343658447,
      "rewards/margins": 8.690082550048828,
      "rewards/rejected": -9.487449645996094,
      "step": 825
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.764452539987179e-05,
      "logits/chosen": -1.6801162958145142,
      "logits/rejected": -1.7072298526763916,
      "logps/chosen": -164.7991943359375,
      "logps/rejected": -236.45445251464844,
      "loss": 0.0628,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.21068036556243896,
      "rewards/margins": 6.561530113220215,
      "rewards/rejected": -6.772209644317627,
      "step": 826
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.761360306830345e-05,
      "logits/chosen": -1.649834394454956,
      "logits/rejected": -1.677789568901062,
      "logps/chosen": -161.15872192382812,
      "logps/rejected": -217.13719177246094,
      "loss": 0.1428,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9511618614196777,
      "rewards/margins": 5.468101978302002,
      "rewards/rejected": -6.4192633628845215,
      "step": 827
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.75826548273127e-05,
      "logits/chosen": -1.6571552753448486,
      "logits/rejected": -1.7653439044952393,
      "logps/chosen": -215.34512329101562,
      "logps/rejected": -283.7718811035156,
      "loss": 0.1087,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.772979199886322,
      "rewards/margins": 5.931948184967041,
      "rewards/rejected": -6.704927921295166,
      "step": 828
    },
    {
      "epoch": 1.08,
      "learning_rate": 3.7551680740469874e-05,
      "logits/chosen": -1.8746492862701416,
      "logits/rejected": -1.846010684967041,
      "logps/chosen": -194.9639129638672,
      "logps/rejected": -278.802001953125,
      "loss": 0.0818,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6157702207565308,
      "rewards/margins": 6.646345615386963,
      "rewards/rejected": -7.262115955352783,
      "step": 829
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.752068087139839e-05,
      "logits/chosen": -1.7149810791015625,
      "logits/rejected": -1.6446678638458252,
      "logps/chosen": -186.3773956298828,
      "logps/rejected": -245.27151489257812,
      "loss": 0.1825,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0201466083526611,
      "rewards/margins": 7.199706554412842,
      "rewards/rejected": -8.219853401184082,
      "step": 830
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.7489655283774657e-05,
      "logits/chosen": -1.9555699825286865,
      "logits/rejected": -2.0322654247283936,
      "logps/chosen": -166.51882934570312,
      "logps/rejected": -217.6027069091797,
      "loss": 0.0656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.890149712562561,
      "rewards/margins": 6.400627136230469,
      "rewards/rejected": -7.29077672958374,
      "step": 831
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.7458604041327874e-05,
      "logits/chosen": -1.8436245918273926,
      "logits/rejected": -1.7562732696533203,
      "logps/chosen": -172.03787231445312,
      "logps/rejected": -267.79901123046875,
      "loss": 0.1362,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0612387657165527,
      "rewards/margins": 5.854422092437744,
      "rewards/rejected": -6.915660858154297,
      "step": 832
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.742752720783997e-05,
      "logits/chosen": -1.9384236335754395,
      "logits/rejected": -1.94955575466156,
      "logps/chosen": -142.2777099609375,
      "logps/rejected": -220.53286743164062,
      "loss": 0.0745,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6656227707862854,
      "rewards/margins": 6.518516540527344,
      "rewards/rejected": -7.184139728546143,
      "step": 833
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.7396424847145425e-05,
      "logits/chosen": -1.6971423625946045,
      "logits/rejected": -1.618373155593872,
      "logps/chosen": -181.0162811279297,
      "logps/rejected": -220.42465209960938,
      "loss": 0.1364,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1203206777572632,
      "rewards/margins": 5.729173183441162,
      "rewards/rejected": -6.849493980407715,
      "step": 834
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.736529702313114e-05,
      "logits/chosen": -2.0127007961273193,
      "logits/rejected": -1.9761312007904053,
      "logps/chosen": -186.38592529296875,
      "logps/rejected": -266.2695617675781,
      "loss": 0.0531,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07478967308998108,
      "rewards/margins": 8.762662887573242,
      "rewards/rejected": -8.687873840332031,
      "step": 835
    },
    {
      "epoch": 1.09,
      "learning_rate": 3.733414379973635e-05,
      "logits/chosen": -1.7820303440093994,
      "logits/rejected": -1.7872676849365234,
      "logps/chosen": -181.1324005126953,
      "logps/rejected": -247.3558349609375,
      "loss": 0.1107,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1160593032836914,
      "rewards/margins": 6.529733180999756,
      "rewards/rejected": -7.6457929611206055,
      "step": 836
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.730296524095245e-05,
      "logits/chosen": -1.7832846641540527,
      "logits/rejected": -1.7420811653137207,
      "logps/chosen": -191.2445831298828,
      "logps/rejected": -225.04779052734375,
      "loss": 0.1667,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5104981660842896,
      "rewards/margins": 4.743764400482178,
      "rewards/rejected": -6.2542619705200195,
      "step": 837
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.7271761410822856e-05,
      "logits/chosen": -1.8028768301010132,
      "logits/rejected": -1.8446567058563232,
      "logps/chosen": -161.13967895507812,
      "logps/rejected": -220.7889404296875,
      "loss": 0.0945,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.173981711268425,
      "rewards/margins": 6.750193119049072,
      "rewards/rejected": -6.924174785614014,
      "step": 838
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.724053237344294e-05,
      "logits/chosen": -1.8580052852630615,
      "logits/rejected": -1.9401862621307373,
      "logps/chosen": -164.49525451660156,
      "logps/rejected": -236.0361328125,
      "loss": 0.0573,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2278103828430176,
      "rewards/margins": 6.170598030090332,
      "rewards/rejected": -7.398408889770508,
      "step": 839
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.720927819295979e-05,
      "logits/chosen": -1.7781224250793457,
      "logits/rejected": -1.7530982494354248,
      "logps/chosen": -175.5201873779297,
      "logps/rejected": -205.05555725097656,
      "loss": 0.0918,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.761866569519043,
      "rewards/margins": 5.51897668838501,
      "rewards/rejected": -6.2808427810668945,
      "step": 840
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.7177998933572186e-05,
      "logits/chosen": -1.849440574645996,
      "logits/rejected": -1.8066624402999878,
      "logps/chosen": -162.9970245361328,
      "logps/rejected": -206.83587646484375,
      "loss": 0.182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9978735446929932,
      "rewards/margins": 4.817579746246338,
      "rewards/rejected": -5.815453052520752,
      "step": 841
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.7146694659530425e-05,
      "logits/chosen": -1.958741307258606,
      "logits/rejected": -1.9996490478515625,
      "logps/chosen": -176.97909545898438,
      "logps/rejected": -224.2952880859375,
      "loss": 0.1747,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3607791066169739,
      "rewards/margins": 6.919254779815674,
      "rewards/rejected": -7.28003454208374,
      "step": 842
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.711536543513614e-05,
      "logits/chosen": -1.615922451019287,
      "logits/rejected": -1.65949285030365,
      "logps/chosen": -136.40432739257812,
      "logps/rejected": -226.84005737304688,
      "loss": 0.1267,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.20042994618415833,
      "rewards/margins": 6.525726318359375,
      "rewards/rejected": -6.726156234741211,
      "step": 843
    },
    {
      "epoch": 1.1,
      "learning_rate": 3.708401132474228e-05,
      "logits/chosen": -1.7548750638961792,
      "logits/rejected": -1.8182183504104614,
      "logps/chosen": -164.22671508789062,
      "logps/rejected": -234.67388916015625,
      "loss": 0.1388,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.38912245631217957,
      "rewards/margins": 6.252824783325195,
      "rewards/rejected": -6.641947269439697,
      "step": 844
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.705263239275284e-05,
      "logits/chosen": -1.981087565422058,
      "logits/rejected": -1.9561117887496948,
      "logps/chosen": -133.49888610839844,
      "logps/rejected": -170.16517639160156,
      "loss": 0.2347,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5047491788864136,
      "rewards/margins": 4.245537757873535,
      "rewards/rejected": -4.7502875328063965,
      "step": 845
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.702122870362286e-05,
      "logits/chosen": -1.9115400314331055,
      "logits/rejected": -1.9423316717147827,
      "logps/chosen": -148.95416259765625,
      "logps/rejected": -224.52301025390625,
      "loss": 0.0484,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36701521277427673,
      "rewards/margins": 6.9237189292907715,
      "rewards/rejected": -7.29073429107666,
      "step": 846
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.698980032185821e-05,
      "logits/chosen": -1.9395225048065186,
      "logits/rejected": -1.9611619710922241,
      "logps/chosen": -142.73492431640625,
      "logps/rejected": -209.59840393066406,
      "loss": 0.0262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4082959294319153,
      "rewards/margins": 5.452874183654785,
      "rewards/rejected": -5.861170768737793,
      "step": 847
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.695834731201548e-05,
      "logits/chosen": -1.6748156547546387,
      "logits/rejected": -1.6887158155441284,
      "logps/chosen": -128.32301330566406,
      "logps/rejected": -196.24961853027344,
      "loss": 0.12,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0047543048858643,
      "rewards/margins": 5.711575508117676,
      "rewards/rejected": -6.716329574584961,
      "step": 848
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.692686973870184e-05,
      "logits/chosen": -1.4874969720840454,
      "logits/rejected": -1.5210838317871094,
      "logps/chosen": -245.0333709716797,
      "logps/rejected": -269.8932800292969,
      "loss": 0.1697,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7569795846939087,
      "rewards/margins": 5.025569438934326,
      "rewards/rejected": -6.782548904418945,
      "step": 849
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.689536766657494e-05,
      "logits/chosen": -1.6894526481628418,
      "logits/rejected": -1.7335946559906006,
      "logps/chosen": -187.63340759277344,
      "logps/rejected": -261.6910400390625,
      "loss": 0.1432,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8881826400756836,
      "rewards/margins": 4.793706893920898,
      "rewards/rejected": -6.681889533996582,
      "step": 850
    },
    {
      "epoch": 1.11,
      "learning_rate": 3.6863841160342723e-05,
      "logits/chosen": -1.9806772470474243,
      "logits/rejected": -1.9756790399551392,
      "logps/chosen": -142.82334899902344,
      "logps/rejected": -207.6903533935547,
      "loss": 0.1081,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2324090003967285,
      "rewards/margins": 5.244317054748535,
      "rewards/rejected": -6.4767255783081055,
      "step": 851
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.683229028476334e-05,
      "logits/chosen": -1.581850528717041,
      "logits/rejected": -1.5908780097961426,
      "logps/chosen": -155.81094360351562,
      "logps/rejected": -206.24839782714844,
      "loss": 0.0606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8023391366004944,
      "rewards/margins": 5.794926166534424,
      "rewards/rejected": -6.597265720367432,
      "step": 852
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.6800715104645e-05,
      "logits/chosen": -1.8793675899505615,
      "logits/rejected": -1.871899127960205,
      "logps/chosen": -151.31143188476562,
      "logps/rejected": -213.91415405273438,
      "loss": 0.0934,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.20195460319519043,
      "rewards/margins": 5.953958511352539,
      "rewards/rejected": -6.15591287612915,
      "step": 853
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.676911568484583e-05,
      "logits/chosen": -1.8364856243133545,
      "logits/rejected": -1.8397459983825684,
      "logps/chosen": -155.30813598632812,
      "logps/rejected": -239.54324340820312,
      "loss": 0.0756,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3419159650802612,
      "rewards/margins": 7.008118629455566,
      "rewards/rejected": -8.350035667419434,
      "step": 854
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.673749209027375e-05,
      "logits/chosen": -1.7932429313659668,
      "logits/rejected": -1.7990435361862183,
      "logps/chosen": -137.10389709472656,
      "logps/rejected": -207.8248291015625,
      "loss": 0.1251,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2713301181793213,
      "rewards/margins": 6.852174282073975,
      "rewards/rejected": -7.123503684997559,
      "step": 855
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.6705844385886334e-05,
      "logits/chosen": -1.965791940689087,
      "logits/rejected": -1.9781205654144287,
      "logps/chosen": -177.0115203857422,
      "logps/rejected": -230.706298828125,
      "loss": 0.0733,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48921334743499756,
      "rewards/margins": 5.834098815917969,
      "rewards/rejected": -6.323312282562256,
      "step": 856
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.667417263669068e-05,
      "logits/chosen": -2.0746617317199707,
      "logits/rejected": -2.110121011734009,
      "logps/chosen": -160.97117614746094,
      "logps/rejected": -228.11575317382812,
      "loss": 0.0222,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32855403423309326,
      "rewards/margins": 6.803640365600586,
      "rewards/rejected": -6.475086212158203,
      "step": 857
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.6642476907743276e-05,
      "logits/chosen": -2.0400307178497314,
      "logits/rejected": -2.042586326599121,
      "logps/chosen": -151.84706115722656,
      "logps/rejected": -205.36061096191406,
      "loss": 0.096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.581629753112793,
      "rewards/margins": 4.815516948699951,
      "rewards/rejected": -5.397147178649902,
      "step": 858
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.661075726414986e-05,
      "logits/chosen": -1.9448158740997314,
      "logits/rejected": -1.9009454250335693,
      "logps/chosen": -154.41232299804688,
      "logps/rejected": -230.76966857910156,
      "loss": 0.0185,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.33493664860725403,
      "rewards/margins": 6.677095890045166,
      "rewards/rejected": -7.012032508850098,
      "step": 859
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.6579013771065305e-05,
      "logits/chosen": -1.6962409019470215,
      "logits/rejected": -1.7169561386108398,
      "logps/chosen": -130.30642700195312,
      "logps/rejected": -224.05807495117188,
      "loss": 0.0538,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.25574082136154175,
      "rewards/margins": 6.733345985412598,
      "rewards/rejected": -6.989086627960205,
      "step": 860
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.654724649369348e-05,
      "logits/chosen": -1.8101693391799927,
      "logits/rejected": -1.84493088722229,
      "logps/chosen": -208.43240356445312,
      "logps/rejected": -260.6061096191406,
      "loss": 0.0705,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2112698256969452,
      "rewards/margins": 6.099943161010742,
      "rewards/rejected": -5.888673305511475,
      "step": 861
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.651545549728709e-05,
      "logits/chosen": -1.7534453868865967,
      "logits/rejected": -1.7846795320510864,
      "logps/chosen": -189.62843322753906,
      "logps/rejected": -241.3184814453125,
      "loss": 0.1383,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5467860102653503,
      "rewards/margins": 5.219336986541748,
      "rewards/rejected": -5.766123294830322,
      "step": 862
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.6483640847147554e-05,
      "logits/chosen": -1.8643182516098022,
      "logits/rejected": -1.9524219036102295,
      "logps/chosen": -177.50558471679688,
      "logps/rejected": -267.6409606933594,
      "loss": 0.0927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.042034849524497986,
      "rewards/margins": 8.53548812866211,
      "rewards/rejected": -8.493453025817871,
      "step": 863
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.645180260862492e-05,
      "logits/chosen": -1.7953535318374634,
      "logits/rejected": -1.9094650745391846,
      "logps/chosen": -147.2042694091797,
      "logps/rejected": -208.8782196044922,
      "loss": 0.1173,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6070430278778076,
      "rewards/margins": 5.45149040222168,
      "rewards/rejected": -6.058534145355225,
      "step": 864
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.6419940847117626e-05,
      "logits/chosen": -1.8601136207580566,
      "logits/rejected": -1.9172056913375854,
      "logps/chosen": -153.92825317382812,
      "logps/rejected": -239.37582397460938,
      "loss": 0.0155,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5362662672996521,
      "rewards/margins": 7.263176441192627,
      "rewards/rejected": -6.726909637451172,
      "step": 865
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.638805562807249e-05,
      "logits/chosen": -2.1471753120422363,
      "logits/rejected": -2.1143898963928223,
      "logps/chosen": -150.82498168945312,
      "logps/rejected": -217.10682678222656,
      "loss": 0.1042,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.35013625025749207,
      "rewards/margins": 6.030050754547119,
      "rewards/rejected": -6.380187034606934,
      "step": 866
    },
    {
      "epoch": 1.13,
      "learning_rate": 3.635614701698448e-05,
      "logits/chosen": -1.8338104486465454,
      "logits/rejected": -1.765840768814087,
      "logps/chosen": -166.82640075683594,
      "logps/rejected": -216.974365234375,
      "loss": 0.1033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1417291164398193,
      "rewards/margins": 4.327848434448242,
      "rewards/rejected": -5.469577789306641,
      "step": 867
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.632421507939661e-05,
      "logits/chosen": -2.05930757522583,
      "logits/rejected": -2.066603899002075,
      "logps/chosen": -162.9051055908203,
      "logps/rejected": -217.9956817626953,
      "loss": 0.0979,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6731106638908386,
      "rewards/margins": 5.83856725692749,
      "rewards/rejected": -6.511678218841553,
      "step": 868
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.629225988089983e-05,
      "logits/chosen": -1.6549468040466309,
      "logits/rejected": -1.6761040687561035,
      "logps/chosen": -126.88896179199219,
      "logps/rejected": -162.6875457763672,
      "loss": 0.1569,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3273538649082184,
      "rewards/margins": 4.158275604248047,
      "rewards/rejected": -4.485629081726074,
      "step": 869
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.6260281487132846e-05,
      "logits/chosen": -2.0777931213378906,
      "logits/rejected": -2.1053049564361572,
      "logps/chosen": -141.18258666992188,
      "logps/rejected": -213.8999786376953,
      "loss": 0.0952,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5611621141433716,
      "rewards/margins": 6.539155006408691,
      "rewards/rejected": -7.100317001342773,
      "step": 870
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.622827996378203e-05,
      "logits/chosen": -1.9073095321655273,
      "logits/rejected": -2.015732765197754,
      "logps/chosen": -133.18026733398438,
      "logps/rejected": -161.3040771484375,
      "loss": 0.1126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.45305973291397095,
      "rewards/margins": 3.4641504287719727,
      "rewards/rejected": -3.917210578918457,
      "step": 871
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.6196255376581254e-05,
      "logits/chosen": -1.976507306098938,
      "logits/rejected": -1.9917340278625488,
      "logps/chosen": -149.463623046875,
      "logps/rejected": -251.50559997558594,
      "loss": 0.01,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2065681666135788,
      "rewards/margins": 7.24117374420166,
      "rewards/rejected": -7.034605503082275,
      "step": 872
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.616420779131177e-05,
      "logits/chosen": -1.90818190574646,
      "logits/rejected": -1.9081482887268066,
      "logps/chosen": -208.44915771484375,
      "logps/rejected": -265.2822265625,
      "loss": 0.2055,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7645907998085022,
      "rewards/margins": 5.729945659637451,
      "rewards/rejected": -6.49453592300415,
      "step": 873
    },
    {
      "epoch": 1.14,
      "learning_rate": 3.613213727380206e-05,
      "logits/chosen": -1.9490686655044556,
      "logits/rejected": -1.9532685279846191,
      "logps/chosen": -151.6713104248047,
      "logps/rejected": -201.02764892578125,
      "loss": 0.0896,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1919515132904053,
      "rewards/margins": 5.8545074462890625,
      "rewards/rejected": -7.046459197998047,
      "step": 874
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.610004388992771e-05,
      "logits/chosen": -1.531323790550232,
      "logits/rejected": -1.5893418788909912,
      "logps/chosen": -119.92091369628906,
      "logps/rejected": -190.87521362304688,
      "loss": 0.0822,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.02865486592054367,
      "rewards/margins": 7.342573165893555,
      "rewards/rejected": -7.3139190673828125,
      "step": 875
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.6067927705611304e-05,
      "logits/chosen": -1.969842553138733,
      "logits/rejected": -1.9398161172866821,
      "logps/chosen": -149.05857849121094,
      "logps/rejected": -244.5399932861328,
      "loss": 0.06,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.039116568863391876,
      "rewards/margins": 6.896425724029541,
      "rewards/rejected": -6.935542583465576,
      "step": 876
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.6035788786822225e-05,
      "logits/chosen": -2.0903704166412354,
      "logits/rejected": -2.1718435287475586,
      "logps/chosen": -186.17471313476562,
      "logps/rejected": -222.77317810058594,
      "loss": 0.1733,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8885395526885986,
      "rewards/margins": 4.886023044586182,
      "rewards/rejected": -5.774562835693359,
      "step": 877
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.6003627199576564e-05,
      "logits/chosen": -1.7578074932098389,
      "logits/rejected": -1.8191397190093994,
      "logps/chosen": -156.69163513183594,
      "logps/rejected": -235.78305053710938,
      "loss": 0.0509,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1328904628753662,
      "rewards/margins": 7.572022438049316,
      "rewards/rejected": -7.704913139343262,
      "step": 878
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.597144300993699e-05,
      "logits/chosen": -1.8367571830749512,
      "logits/rejected": -1.9178485870361328,
      "logps/chosen": -203.2810516357422,
      "logps/rejected": -253.2967987060547,
      "loss": 0.1422,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8315750360488892,
      "rewards/margins": 5.403246879577637,
      "rewards/rejected": -6.2348222732543945,
      "step": 879
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.593923628401259e-05,
      "logits/chosen": -2.070420265197754,
      "logits/rejected": -2.117854356765747,
      "logps/chosen": -129.53744506835938,
      "logps/rejected": -213.93307495117188,
      "loss": 0.1386,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.20923367142677307,
      "rewards/margins": 7.3108906745910645,
      "rewards/rejected": -7.101656436920166,
      "step": 880
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.5907007087958726e-05,
      "logits/chosen": -1.6655091047286987,
      "logits/rejected": -1.693989634513855,
      "logps/chosen": -168.85360717773438,
      "logps/rejected": -233.3699951171875,
      "loss": 0.2251,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.492025375366211,
      "rewards/margins": 4.709729194641113,
      "rewards/rejected": -6.201755046844482,
      "step": 881
    },
    {
      "epoch": 1.15,
      "learning_rate": 3.587475548797694e-05,
      "logits/chosen": -2.0595686435699463,
      "logits/rejected": -1.963339924812317,
      "logps/chosen": -149.55838012695312,
      "logps/rejected": -191.93299865722656,
      "loss": 0.1129,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6186510920524597,
      "rewards/margins": 5.05853271484375,
      "rewards/rejected": -5.677184104919434,
      "step": 882
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.5842481550314794e-05,
      "logits/chosen": -1.9392576217651367,
      "logits/rejected": -1.889467477798462,
      "logps/chosen": -168.27606201171875,
      "logps/rejected": -279.1664733886719,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9032707214355469,
      "rewards/margins": 8.34951400756836,
      "rewards/rejected": -9.25278377532959,
      "step": 883
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.581018534126571e-05,
      "logits/chosen": -2.025470495223999,
      "logits/rejected": -2.0483429431915283,
      "logps/chosen": -201.74916076660156,
      "logps/rejected": -267.3995361328125,
      "loss": 0.08,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8922535181045532,
      "rewards/margins": 5.836456775665283,
      "rewards/rejected": -6.728710174560547,
      "step": 884
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.577786692716886e-05,
      "logits/chosen": -1.8123650550842285,
      "logits/rejected": -1.8336703777313232,
      "logps/chosen": -166.75900268554688,
      "logps/rejected": -210.2139129638672,
      "loss": 0.107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.29484596848487854,
      "rewards/margins": 5.857831001281738,
      "rewards/rejected": -6.152677059173584,
      "step": 885
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.574552637440907e-05,
      "logits/chosen": -1.826763391494751,
      "logits/rejected": -1.8925033807754517,
      "logps/chosen": -147.36468505859375,
      "logps/rejected": -225.35337829589844,
      "loss": 0.0689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.057733140885829926,
      "rewards/margins": 6.5763630867004395,
      "rewards/rejected": -6.634096622467041,
      "step": 886
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.571316374941658e-05,
      "logits/chosen": -1.8496880531311035,
      "logits/rejected": -1.9214247465133667,
      "logps/chosen": -143.42526245117188,
      "logps/rejected": -276.57928466796875,
      "loss": 0.0297,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7962692975997925,
      "rewards/margins": 8.667549133300781,
      "rewards/rejected": -9.46381950378418,
      "step": 887
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.568077911866703e-05,
      "logits/chosen": -2.2294137477874756,
      "logits/rejected": -2.256291627883911,
      "logps/chosen": -165.8625030517578,
      "logps/rejected": -226.53073120117188,
      "loss": 0.1032,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0509380102157593,
      "rewards/margins": 5.621366500854492,
      "rewards/rejected": -6.672304153442383,
      "step": 888
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.564837254868118e-05,
      "logits/chosen": -1.8474410772323608,
      "logits/rejected": -1.8447623252868652,
      "logps/chosen": -170.68482971191406,
      "logps/rejected": -241.2418212890625,
      "loss": 0.1629,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5344374179840088,
      "rewards/margins": 6.09687614440918,
      "rewards/rejected": -6.631314277648926,
      "step": 889
    },
    {
      "epoch": 1.16,
      "learning_rate": 3.561594410602495e-05,
      "logits/chosen": -1.969462513923645,
      "logits/rejected": -1.974022388458252,
      "logps/chosen": -155.7165985107422,
      "logps/rejected": -204.6715087890625,
      "loss": 0.1178,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.070492148399353,
      "rewards/margins": 5.344703197479248,
      "rewards/rejected": -6.415195465087891,
      "step": 890
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.558349385730913e-05,
      "logits/chosen": -1.9857453107833862,
      "logits/rejected": -1.9735053777694702,
      "logps/chosen": -171.04324340820312,
      "logps/rejected": -229.79617309570312,
      "loss": 0.0638,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3228389322757721,
      "rewards/margins": 6.23259162902832,
      "rewards/rejected": -6.555430889129639,
      "step": 891
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.5551021869189286e-05,
      "logits/chosen": -1.9572652578353882,
      "logits/rejected": -2.0211684703826904,
      "logps/chosen": -177.00323486328125,
      "logps/rejected": -242.7465362548828,
      "loss": 0.0911,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09810394793748856,
      "rewards/margins": 6.194761753082275,
      "rewards/rejected": -6.292865753173828,
      "step": 892
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.55185282083657e-05,
      "logits/chosen": -2.00260066986084,
      "logits/rejected": -2.0481128692626953,
      "logps/chosen": -179.40768432617188,
      "logps/rejected": -237.79440307617188,
      "loss": 0.1789,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9579204320907593,
      "rewards/margins": 5.24198579788208,
      "rewards/rejected": -6.1999053955078125,
      "step": 893
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.548601294158313e-05,
      "logits/chosen": -1.9575377702713013,
      "logits/rejected": -1.9701429605484009,
      "logps/chosen": -170.8883819580078,
      "logps/rejected": -204.66455078125,
      "loss": 0.2138,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2080439329147339,
      "rewards/margins": 3.7738595008850098,
      "rewards/rejected": -4.981903076171875,
      "step": 894
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.5453476135630706e-05,
      "logits/chosen": -2.005340814590454,
      "logits/rejected": -2.003964900970459,
      "logps/chosen": -153.75393676757812,
      "logps/rejected": -222.0598907470703,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16159169375896454,
      "rewards/margins": 6.470172882080078,
      "rewards/rejected": -6.308581352233887,
      "step": 895
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.542091785734184e-05,
      "logits/chosen": -1.9905329942703247,
      "logits/rejected": -1.9771034717559814,
      "logps/chosen": -140.4086456298828,
      "logps/rejected": -171.7359619140625,
      "loss": 0.1531,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.499469518661499,
      "rewards/margins": 4.087397575378418,
      "rewards/rejected": -4.586867332458496,
      "step": 896
    },
    {
      "epoch": 1.17,
      "learning_rate": 3.538833817359401e-05,
      "logits/chosen": -2.136502265930176,
      "logits/rejected": -2.1341865062713623,
      "logps/chosen": -172.52622985839844,
      "logps/rejected": -240.81138610839844,
      "loss": 0.188,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5810390114784241,
      "rewards/margins": 6.328227996826172,
      "rewards/rejected": -6.909266948699951,
      "step": 897
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.5355737151308686e-05,
      "logits/chosen": -1.9634038209915161,
      "logits/rejected": -1.9026023149490356,
      "logps/chosen": -133.00393676757812,
      "logps/rejected": -188.03909301757812,
      "loss": 0.0604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5214068293571472,
      "rewards/margins": 5.715162754058838,
      "rewards/rejected": -6.236569881439209,
      "step": 898
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.5323114857451174e-05,
      "logits/chosen": -2.059035301208496,
      "logits/rejected": -2.093261241912842,
      "logps/chosen": -178.64810180664062,
      "logps/rejected": -254.74835205078125,
      "loss": 0.0499,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.20145034790039062,
      "rewards/margins": 7.516545295715332,
      "rewards/rejected": -7.7179951667785645,
      "step": 899
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.529047135903045e-05,
      "logits/chosen": -1.9401965141296387,
      "logits/rejected": -1.893580675125122,
      "logps/chosen": -148.152099609375,
      "logps/rejected": -217.60165405273438,
      "loss": 0.0832,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22576339542865753,
      "rewards/margins": 6.487100124359131,
      "rewards/rejected": -6.2613372802734375,
      "step": 900
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.525780672309907e-05,
      "logits/chosen": -2.099334478378296,
      "logits/rejected": -2.1224048137664795,
      "logps/chosen": -176.6794891357422,
      "logps/rejected": -189.8108367919922,
      "loss": 0.1743,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3507277965545654,
      "rewards/margins": 4.109447956085205,
      "rewards/rejected": -5.460175514221191,
      "step": 901
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.522512101675299e-05,
      "logits/chosen": -2.1799097061157227,
      "logits/rejected": -2.176818609237671,
      "logps/chosen": -181.85447692871094,
      "logps/rejected": -213.69349670410156,
      "loss": 0.2637,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0590920448303223,
      "rewards/margins": 3.6977596282958984,
      "rewards/rejected": -4.756852149963379,
      "step": 902
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.519241430713145e-05,
      "logits/chosen": -1.9091711044311523,
      "logits/rejected": -2.006434679031372,
      "logps/chosen": -143.8796844482422,
      "logps/rejected": -204.40049743652344,
      "loss": 0.0631,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.471727192401886,
      "rewards/margins": 6.225522994995117,
      "rewards/rejected": -6.6972503662109375,
      "step": 903
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.5159686661416834e-05,
      "logits/chosen": -1.8754589557647705,
      "logits/rejected": -1.8962466716766357,
      "logps/chosen": -160.32774353027344,
      "logps/rejected": -256.0925598144531,
      "loss": 0.0538,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6474647521972656,
      "rewards/margins": 8.686860084533691,
      "rewards/rejected": -8.039395332336426,
      "step": 904
    },
    {
      "epoch": 1.18,
      "learning_rate": 3.512693814683456e-05,
      "logits/chosen": -1.6069674491882324,
      "logits/rejected": -1.6205403804779053,
      "logps/chosen": -159.8393096923828,
      "logps/rejected": -223.45440673828125,
      "loss": 0.107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5345543622970581,
      "rewards/margins": 4.439685344696045,
      "rewards/rejected": -4.974239826202393,
      "step": 905
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.5094168830652854e-05,
      "logits/chosen": -1.88560950756073,
      "logits/rejected": -1.9885283708572388,
      "logps/chosen": -162.08815002441406,
      "logps/rejected": -214.88351440429688,
      "loss": 0.1758,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.36401665210723877,
      "rewards/margins": 5.732705116271973,
      "rewards/rejected": -6.09672212600708,
      "step": 906
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.506137878018272e-05,
      "logits/chosen": -2.0618391036987305,
      "logits/rejected": -2.058894634246826,
      "logps/chosen": -158.00828552246094,
      "logps/rejected": -212.74961853027344,
      "loss": 0.0398,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6322505474090576,
      "rewards/margins": 5.919046401977539,
      "rewards/rejected": -6.551296710968018,
      "step": 907
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.502856806277773e-05,
      "logits/chosen": -1.9531464576721191,
      "logits/rejected": -2.014647960662842,
      "logps/chosen": -155.9281005859375,
      "logps/rejected": -252.1271209716797,
      "loss": 0.0108,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32207781076431274,
      "rewards/margins": 9.11711597442627,
      "rewards/rejected": -8.795039176940918,
      "step": 908
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.4995736745833895e-05,
      "logits/chosen": -2.0440237522125244,
      "logits/rejected": -2.0321359634399414,
      "logps/chosen": -193.749267578125,
      "logps/rejected": -250.82113647460938,
      "loss": 0.122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3427705764770508,
      "rewards/margins": 5.0545477867126465,
      "rewards/rejected": -5.3973188400268555,
      "step": 909
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.496288489678958e-05,
      "logits/chosen": -2.010633707046509,
      "logits/rejected": -1.9929156303405762,
      "logps/chosen": -166.10728454589844,
      "logps/rejected": -248.0762939453125,
      "loss": 0.0532,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27336451411247253,
      "rewards/margins": 7.333755016326904,
      "rewards/rejected": -7.060391426086426,
      "step": 910
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.493001258312529e-05,
      "logits/chosen": -2.1049704551696777,
      "logits/rejected": -2.184370279312134,
      "logps/chosen": -152.46676635742188,
      "logps/rejected": -193.95933532714844,
      "loss": 0.1413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22922754287719727,
      "rewards/margins": 5.094156265258789,
      "rewards/rejected": -5.323383331298828,
      "step": 911
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.489711987236357e-05,
      "logits/chosen": -1.8540668487548828,
      "logits/rejected": -1.8573285341262817,
      "logps/chosen": -150.3858642578125,
      "logps/rejected": -199.2803192138672,
      "loss": 0.1095,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.062329575419425964,
      "rewards/margins": 4.983062744140625,
      "rewards/rejected": -4.920733451843262,
      "step": 912
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.4864206832068884e-05,
      "logits/chosen": -1.9019203186035156,
      "logits/rejected": -1.9186983108520508,
      "logps/chosen": -166.6719207763672,
      "logps/rejected": -236.49954223632812,
      "loss": 0.059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6351613402366638,
      "rewards/margins": 5.781877040863037,
      "rewards/rejected": -6.417038440704346,
      "step": 913
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.483127352984742e-05,
      "logits/chosen": -1.8288668394088745,
      "logits/rejected": -1.8692231178283691,
      "logps/chosen": -203.69033813476562,
      "logps/rejected": -294.76300048828125,
      "loss": 0.1123,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3427416682243347,
      "rewards/margins": 6.824990272521973,
      "rewards/rejected": -6.482248306274414,
      "step": 914
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.479832003334702e-05,
      "logits/chosen": -1.9807475805282593,
      "logits/rejected": -1.9427908658981323,
      "logps/chosen": -154.55177307128906,
      "logps/rejected": -220.6343536376953,
      "loss": 0.0636,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.33972498774528503,
      "rewards/margins": 6.373534202575684,
      "rewards/rejected": -6.033810615539551,
      "step": 915
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.476534641025698e-05,
      "logits/chosen": -1.9640017747879028,
      "logits/rejected": -2.0024826526641846,
      "logps/chosen": -144.81207275390625,
      "logps/rejected": -178.35458374023438,
      "loss": 0.106,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5719605088233948,
      "rewards/margins": 4.674602031707764,
      "rewards/rejected": -5.2465620040893555,
      "step": 916
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.4732352728307966e-05,
      "logits/chosen": -2.039835214614868,
      "logits/rejected": -2.054598569869995,
      "logps/chosen": -137.9036407470703,
      "logps/rejected": -193.40792846679688,
      "loss": 0.171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.238839864730835,
      "rewards/margins": 4.93145751953125,
      "rewards/rejected": -6.170297622680664,
      "step": 917
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.469933905527182e-05,
      "logits/chosen": -2.0529818534851074,
      "logits/rejected": -2.0553789138793945,
      "logps/chosen": -181.83724975585938,
      "logps/rejected": -250.62246704101562,
      "loss": 0.0482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.027744874358177185,
      "rewards/margins": 7.317486763000488,
      "rewards/rejected": -7.345231056213379,
      "step": 918
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.466630545896146e-05,
      "logits/chosen": -1.7903227806091309,
      "logits/rejected": -1.8518483638763428,
      "logps/chosen": -148.36544799804688,
      "logps/rejected": -238.90078735351562,
      "loss": 0.0195,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14351078867912292,
      "rewards/margins": 6.769139289855957,
      "rewards/rejected": -6.912649631500244,
      "step": 919
    },
    {
      "epoch": 1.2,
      "learning_rate": 3.463325200723071e-05,
      "logits/chosen": -1.8101056814193726,
      "logits/rejected": -1.8288451433181763,
      "logps/chosen": -129.47695922851562,
      "logps/rejected": -183.96656799316406,
      "loss": 0.061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39186879992485046,
      "rewards/margins": 5.672905921936035,
      "rewards/rejected": -6.064774036407471,
      "step": 920
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.460017876797422e-05,
      "logits/chosen": -1.7113351821899414,
      "logits/rejected": -1.7809813022613525,
      "logps/chosen": -167.154541015625,
      "logps/rejected": -234.83380126953125,
      "loss": 0.0302,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08966308832168579,
      "rewards/margins": 7.432629108428955,
      "rewards/rejected": -7.342965602874756,
      "step": 921
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.456708580912725e-05,
      "logits/chosen": -1.7443047761917114,
      "logits/rejected": -1.7694761753082275,
      "logps/chosen": -139.82110595703125,
      "logps/rejected": -190.7325439453125,
      "loss": 0.1295,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8659021854400635,
      "rewards/margins": 5.205358982086182,
      "rewards/rejected": -6.071261882781982,
      "step": 922
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.453397319866557e-05,
      "logits/chosen": -2.0398495197296143,
      "logits/rejected": -2.0124411582946777,
      "logps/chosen": -151.38482666015625,
      "logps/rejected": -214.8488311767578,
      "loss": 0.193,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8532832860946655,
      "rewards/margins": 5.717514991760254,
      "rewards/rejected": -6.570797920227051,
      "step": 923
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.4500841004605324e-05,
      "logits/chosen": -2.002171039581299,
      "logits/rejected": -2.0679383277893066,
      "logps/chosen": -218.12924194335938,
      "logps/rejected": -283.44219970703125,
      "loss": 0.0952,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.25244951248168945,
      "rewards/margins": 5.972373008728027,
      "rewards/rejected": -6.224822044372559,
      "step": 924
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.446768929500288e-05,
      "logits/chosen": -1.782718539237976,
      "logits/rejected": -1.842342734336853,
      "logps/chosen": -170.8223876953125,
      "logps/rejected": -227.46841430664062,
      "loss": 0.1333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4326009750366211,
      "rewards/margins": 6.7193145751953125,
      "rewards/rejected": -7.151915550231934,
      "step": 925
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.443451813795469e-05,
      "logits/chosen": -1.9137529134750366,
      "logits/rejected": -1.9013252258300781,
      "logps/chosen": -166.90280151367188,
      "logps/rejected": -216.41151428222656,
      "loss": 0.172,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.974904477596283,
      "rewards/margins": 5.321615695953369,
      "rewards/rejected": -6.296520709991455,
      "step": 926
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.4401327601597174e-05,
      "logits/chosen": -2.1041436195373535,
      "logits/rejected": -2.086627721786499,
      "logps/chosen": -140.56207275390625,
      "logps/rejected": -184.04986572265625,
      "loss": 0.1364,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7975207567214966,
      "rewards/margins": 5.4658427238464355,
      "rewards/rejected": -6.263363361358643,
      "step": 927
    },
    {
      "epoch": 1.21,
      "learning_rate": 3.436811775410651e-05,
      "logits/chosen": -2.108426570892334,
      "logits/rejected": -2.1942784786224365,
      "logps/chosen": -189.42230224609375,
      "logps/rejected": -268.1767578125,
      "loss": 0.118,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.321402907371521,
      "rewards/margins": 5.401063442230225,
      "rewards/rejected": -6.722466468811035,
      "step": 928
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.43348886636986e-05,
      "logits/chosen": -1.9828169345855713,
      "logits/rejected": -1.994112491607666,
      "logps/chosen": -151.9613494873047,
      "logps/rejected": -221.04864501953125,
      "loss": 0.0853,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6451107859611511,
      "rewards/margins": 5.837140083312988,
      "rewards/rejected": -6.482250690460205,
      "step": 929
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.430164039862882e-05,
      "logits/chosen": -1.779541254043579,
      "logits/rejected": -1.7532923221588135,
      "logps/chosen": -202.18359375,
      "logps/rejected": -259.9750061035156,
      "loss": 0.1775,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7676839828491211,
      "rewards/margins": 5.355978012084961,
      "rewards/rejected": -6.12366247177124,
      "step": 930
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.426837302719197e-05,
      "logits/chosen": -1.8977571725845337,
      "logits/rejected": -1.91450035572052,
      "logps/chosen": -161.89410400390625,
      "logps/rejected": -217.23263549804688,
      "loss": 0.1046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7960255742073059,
      "rewards/margins": 5.442742824554443,
      "rewards/rejected": -6.238769054412842,
      "step": 931
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.42350866177221e-05,
      "logits/chosen": -1.796115517616272,
      "logits/rejected": -1.8026353120803833,
      "logps/chosen": -167.72669982910156,
      "logps/rejected": -235.31195068359375,
      "loss": 0.0101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7101820111274719,
      "rewards/margins": 7.052955150604248,
      "rewards/rejected": -7.763136386871338,
      "step": 932
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.420178123859233e-05,
      "logits/chosen": -1.884534239768982,
      "logits/rejected": -1.9360039234161377,
      "logps/chosen": -175.97605895996094,
      "logps/rejected": -280.8975524902344,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3528714179992676,
      "rewards/margins": 7.842462539672852,
      "rewards/rejected": -8.195335388183594,
      "step": 933
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.416845695821476e-05,
      "logits/chosen": -1.6204770803451538,
      "logits/rejected": -1.7952289581298828,
      "logps/chosen": -141.75640869140625,
      "logps/rejected": -205.68414306640625,
      "loss": 0.0849,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3700450658798218,
      "rewards/margins": 5.952082633972168,
      "rewards/rejected": -7.322127342224121,
      "step": 934
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.413511384504034e-05,
      "logits/chosen": -1.999087929725647,
      "logits/rejected": -2.0146172046661377,
      "logps/chosen": -181.1517791748047,
      "logps/rejected": -228.80722045898438,
      "loss": 0.0826,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20987439155578613,
      "rewards/margins": 6.5635085105896,
      "rewards/rejected": -6.353633403778076,
      "step": 935
    },
    {
      "epoch": 1.22,
      "learning_rate": 3.410175196755866e-05,
      "logits/chosen": -1.9721521139144897,
      "logits/rejected": -2.0418949127197266,
      "logps/chosen": -205.38955688476562,
      "logps/rejected": -272.0664367675781,
      "loss": 0.1197,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2950458526611328,
      "rewards/margins": 5.9981231689453125,
      "rewards/rejected": -7.2931694984436035,
      "step": 936
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.40683713942979e-05,
      "logits/chosen": -2.0940282344818115,
      "logits/rejected": -2.13757586479187,
      "logps/chosen": -155.8927459716797,
      "logps/rejected": -231.48611450195312,
      "loss": 0.0444,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1400851011276245,
      "rewards/margins": 6.619723320007324,
      "rewards/rejected": -7.75980806350708,
      "step": 937
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.403497219382461e-05,
      "logits/chosen": -1.7476333379745483,
      "logits/rejected": -1.7370529174804688,
      "logps/chosen": -148.12130737304688,
      "logps/rejected": -232.86611938476562,
      "loss": 0.1126,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08682404458522797,
      "rewards/margins": 7.703474044799805,
      "rewards/rejected": -7.790297985076904,
      "step": 938
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.400155443474361e-05,
      "logits/chosen": -2.0138838291168213,
      "logits/rejected": -2.05570912361145,
      "logps/chosen": -162.871826171875,
      "logps/rejected": -249.21856689453125,
      "loss": 0.1125,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1651180535554886,
      "rewards/margins": 7.845034122467041,
      "rewards/rejected": -8.010151863098145,
      "step": 939
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.396811818569785e-05,
      "logits/chosen": -1.6162145137786865,
      "logits/rejected": -1.7617723941802979,
      "logps/chosen": -154.22023010253906,
      "logps/rejected": -233.10836791992188,
      "loss": 0.1039,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.20926518738269806,
      "rewards/margins": 6.4955153465271,
      "rewards/rejected": -6.7047810554504395,
      "step": 940
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.3934663515368236e-05,
      "logits/chosen": -2.0607364177703857,
      "logits/rejected": -2.064283847808838,
      "logps/chosen": -181.87879943847656,
      "logps/rejected": -239.00088500976562,
      "loss": 0.0548,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.947770357131958,
      "rewards/margins": 5.138289451599121,
      "rewards/rejected": -7.086060047149658,
      "step": 941
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.3901190492473554e-05,
      "logits/chosen": -2.0121662616729736,
      "logits/rejected": -1.9954758882522583,
      "logps/chosen": -164.00282287597656,
      "logps/rejected": -251.4072723388672,
      "loss": 0.057,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6742583513259888,
      "rewards/margins": 6.951840877532959,
      "rewards/rejected": -7.626099109649658,
      "step": 942
    },
    {
      "epoch": 1.23,
      "learning_rate": 3.3867699185770255e-05,
      "logits/chosen": -1.8472353219985962,
      "logits/rejected": -1.953643560409546,
      "logps/chosen": -150.764404296875,
      "logps/rejected": -238.75494384765625,
      "loss": 0.0171,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6845617294311523,
      "rewards/margins": 6.793731689453125,
      "rewards/rejected": -7.478293418884277,
      "step": 943
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.383418966405234e-05,
      "logits/chosen": -2.131664276123047,
      "logits/rejected": -2.15356707572937,
      "logps/chosen": -174.39974975585938,
      "logps/rejected": -247.7554168701172,
      "loss": 0.0517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.35220858454704285,
      "rewards/margins": 7.23449182510376,
      "rewards/rejected": -7.586700439453125,
      "step": 944
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.3800661996151264e-05,
      "logits/chosen": -2.0658442974090576,
      "logits/rejected": -2.0301132202148438,
      "logps/chosen": -163.01617431640625,
      "logps/rejected": -198.7908477783203,
      "loss": 0.1661,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4769498109817505,
      "rewards/margins": 4.503304481506348,
      "rewards/rejected": -5.980254173278809,
      "step": 945
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.376711625093571e-05,
      "logits/chosen": -1.9626531600952148,
      "logits/rejected": -2.005638360977173,
      "logps/chosen": -227.94239807128906,
      "logps/rejected": -285.3360900878906,
      "loss": 0.121,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0329959392547607,
      "rewards/margins": 5.07936429977417,
      "rewards/rejected": -6.11236047744751,
      "step": 946
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.373355249731153e-05,
      "logits/chosen": -1.6072015762329102,
      "logits/rejected": -1.6854274272918701,
      "logps/chosen": -149.63619995117188,
      "logps/rejected": -231.1209716796875,
      "loss": 0.0537,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.43269726634025574,
      "rewards/margins": 7.149322032928467,
      "rewards/rejected": -7.582018852233887,
      "step": 947
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.369997080422155e-05,
      "logits/chosen": -1.702265739440918,
      "logits/rejected": -1.7799084186553955,
      "logps/chosen": -170.57896423339844,
      "logps/rejected": -253.85064697265625,
      "loss": 0.146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.236419916152954,
      "rewards/margins": 6.673525810241699,
      "rewards/rejected": -7.909945964813232,
      "step": 948
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.366637124064544e-05,
      "logits/chosen": -1.895546793937683,
      "logits/rejected": -1.9188759326934814,
      "logps/chosen": -179.75929260253906,
      "logps/rejected": -222.67404174804688,
      "loss": 0.1123,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3694798946380615,
      "rewards/margins": 5.722755432128906,
      "rewards/rejected": -7.092235088348389,
      "step": 949
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.36327538755996e-05,
      "logits/chosen": -1.7629483938217163,
      "logits/rejected": -1.834211826324463,
      "logps/chosen": -163.36801147460938,
      "logps/rejected": -240.06100463867188,
      "loss": 0.0835,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1130611896514893,
      "rewards/margins": 6.393387794494629,
      "rewards/rejected": -7.5064496994018555,
      "step": 950
    },
    {
      "epoch": 1.24,
      "learning_rate": 3.3599118778136965e-05,
      "logits/chosen": -1.956146478652954,
      "logits/rejected": -1.949027180671692,
      "logps/chosen": -154.01885986328125,
      "logps/rejected": -214.5371856689453,
      "loss": 0.0942,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0469669103622437,
      "rewards/margins": 5.84091329574585,
      "rewards/rejected": -6.887880325317383,
      "step": 951
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.356546601734692e-05,
      "logits/chosen": -2.103282928466797,
      "logits/rejected": -2.1140389442443848,
      "logps/chosen": -160.41705322265625,
      "logps/rejected": -222.67813110351562,
      "loss": 0.0934,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0452954769134521,
      "rewards/margins": 5.0740461349487305,
      "rewards/rejected": -6.119342803955078,
      "step": 952
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.3531795662355115e-05,
      "logits/chosen": -2.031980276107788,
      "logits/rejected": -2.070544481277466,
      "logps/chosen": -160.99761962890625,
      "logps/rejected": -243.9342041015625,
      "loss": 0.0933,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9394694566726685,
      "rewards/margins": 7.686059951782227,
      "rewards/rejected": -8.625529289245605,
      "step": 953
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.349810778232335e-05,
      "logits/chosen": -1.8342362642288208,
      "logits/rejected": -1.9654871225357056,
      "logps/chosen": -167.48745727539062,
      "logps/rejected": -208.47169494628906,
      "loss": 0.1846,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0384410619735718,
      "rewards/margins": 4.180461883544922,
      "rewards/rejected": -5.218903064727783,
      "step": 954
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.346440244644942e-05,
      "logits/chosen": -1.765977144241333,
      "logits/rejected": -1.7814247608184814,
      "logps/chosen": -162.0328826904297,
      "logps/rejected": -251.04905700683594,
      "loss": 0.0176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1184916496276855,
      "rewards/margins": 6.529496669769287,
      "rewards/rejected": -7.647988319396973,
      "step": 955
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.3430679723966976e-05,
      "logits/chosen": -1.7043746709823608,
      "logits/rejected": -1.6794589757919312,
      "logps/chosen": -177.63613891601562,
      "logps/rejected": -251.107666015625,
      "loss": 0.1218,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0984007120132446,
      "rewards/margins": 7.543315410614014,
      "rewards/rejected": -8.641716003417969,
      "step": 956
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.339693968414538e-05,
      "logits/chosen": -1.9795281887054443,
      "logits/rejected": -1.9849295616149902,
      "logps/chosen": -137.0846710205078,
      "logps/rejected": -184.3840789794922,
      "loss": 0.2054,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7593737840652466,
      "rewards/margins": 4.5130181312561035,
      "rewards/rejected": -5.2723917961120605,
      "step": 957
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.336318239628956e-05,
      "logits/chosen": -2.0036728382110596,
      "logits/rejected": -1.933376669883728,
      "logps/chosen": -211.4518585205078,
      "logps/rejected": -290.0897216796875,
      "loss": 0.1094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8515461087226868,
      "rewards/margins": 5.591423511505127,
      "rewards/rejected": -6.442969799041748,
      "step": 958
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.3329407929739906e-05,
      "logits/chosen": -1.83505117893219,
      "logits/rejected": -1.8577007055282593,
      "logps/chosen": -186.45472717285156,
      "logps/rejected": -260.4599304199219,
      "loss": 0.067,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8454413414001465,
      "rewards/margins": 6.24069881439209,
      "rewards/rejected": -7.0861406326293945,
      "step": 959
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.3295616353872026e-05,
      "logits/chosen": -1.5665518045425415,
      "logits/rejected": -1.5015395879745483,
      "logps/chosen": -196.26315307617188,
      "logps/rejected": -232.317138671875,
      "loss": 0.0214,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3765389919281006,
      "rewards/margins": 5.861631393432617,
      "rewards/rejected": -7.238170623779297,
      "step": 960
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.326180773809676e-05,
      "logits/chosen": -1.9945480823516846,
      "logits/rejected": -2.0716819763183594,
      "logps/chosen": -181.3908233642578,
      "logps/rejected": -261.51983642578125,
      "loss": 0.1299,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5620561838150024,
      "rewards/margins": 5.328022480010986,
      "rewards/rejected": -6.890079021453857,
      "step": 961
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.3227982151859873e-05,
      "logits/chosen": -2.0687661170959473,
      "logits/rejected": -2.008908271789551,
      "logps/chosen": -188.9337615966797,
      "logps/rejected": -245.06329345703125,
      "loss": 0.058,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7507072687149048,
      "rewards/margins": 5.501278400421143,
      "rewards/rejected": -6.251986026763916,
      "step": 962
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.3194139664642035e-05,
      "logits/chosen": -2.0266988277435303,
      "logits/rejected": -2.055248260498047,
      "logps/chosen": -170.5966339111328,
      "logps/rejected": -261.96221923828125,
      "loss": 0.1398,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.17907534539699554,
      "rewards/margins": 7.6258544921875,
      "rewards/rejected": -7.804930686950684,
      "step": 963
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.3160280345958614e-05,
      "logits/chosen": -2.056675434112549,
      "logits/rejected": -2.0761566162109375,
      "logps/chosen": -147.96145629882812,
      "logps/rejected": -208.98709106445312,
      "loss": 0.1157,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7914592623710632,
      "rewards/margins": 5.899379730224609,
      "rewards/rejected": -6.690839767456055,
      "step": 964
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.3126404265359545e-05,
      "logits/chosen": -2.066800832748413,
      "logits/rejected": -2.0727744102478027,
      "logps/chosen": -142.2117156982422,
      "logps/rejected": -224.7412872314453,
      "loss": 0.0703,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2246077060699463,
      "rewards/margins": 5.702795505523682,
      "rewards/rejected": -6.927403450012207,
      "step": 965
    },
    {
      "epoch": 1.26,
      "learning_rate": 3.3092511492429216e-05,
      "logits/chosen": -1.8461310863494873,
      "logits/rejected": -1.9243555068969727,
      "logps/chosen": -137.1815643310547,
      "logps/rejected": -208.0272216796875,
      "loss": 0.0951,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0130833387374878,
      "rewards/margins": 5.715978622436523,
      "rewards/rejected": -6.729062080383301,
      "step": 966
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.305860209678628e-05,
      "logits/chosen": -1.9411567449569702,
      "logits/rejected": -1.9278594255447388,
      "logps/chosen": -136.62242126464844,
      "logps/rejected": -198.45223999023438,
      "loss": 0.1076,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.49253129959106445,
      "rewards/margins": 5.900206089019775,
      "rewards/rejected": -6.39273738861084,
      "step": 967
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.3024676148083555e-05,
      "logits/chosen": -1.78202223777771,
      "logits/rejected": -1.7654985189437866,
      "logps/chosen": -193.89064025878906,
      "logps/rejected": -248.55442810058594,
      "loss": 0.1166,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5818485021591187,
      "rewards/margins": 6.548543930053711,
      "rewards/rejected": -7.130392551422119,
      "step": 968
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.299073371600784e-05,
      "logits/chosen": -1.9086130857467651,
      "logits/rejected": -1.8986905813217163,
      "logps/chosen": -153.02613830566406,
      "logps/rejected": -183.6855926513672,
      "loss": 0.0734,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0142678022384644,
      "rewards/margins": 4.934293746948242,
      "rewards/rejected": -5.948561668395996,
      "step": 969
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.29567748702798e-05,
      "logits/chosen": -2.012598991394043,
      "logits/rejected": -2.0287890434265137,
      "logps/chosen": -213.5174102783203,
      "logps/rejected": -255.91893005371094,
      "loss": 0.1708,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9519654512405396,
      "rewards/margins": 4.617500305175781,
      "rewards/rejected": -5.569465637207031,
      "step": 970
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.2922799680653816e-05,
      "logits/chosen": -1.865847110748291,
      "logits/rejected": -1.846561312675476,
      "logps/chosen": -210.79624938964844,
      "logps/rejected": -282.8587341308594,
      "loss": 0.1333,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.6331790685653687,
      "rewards/margins": 7.995599746704102,
      "rewards/rejected": -7.362420082092285,
      "step": 971
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.288880821691785e-05,
      "logits/chosen": -1.8660513162612915,
      "logits/rejected": -1.8335342407226562,
      "logps/chosen": -171.3505096435547,
      "logps/rejected": -233.75755310058594,
      "loss": 0.0517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.488456130027771,
      "rewards/margins": 6.7542009353637695,
      "rewards/rejected": -7.242657661437988,
      "step": 972
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.285480054889327e-05,
      "logits/chosen": -1.8647754192352295,
      "logits/rejected": -1.8496925830841064,
      "logps/chosen": -160.0685577392578,
      "logps/rejected": -222.1055145263672,
      "loss": 0.1012,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09002574533224106,
      "rewards/margins": 6.101812362670898,
      "rewards/rejected": -6.011786460876465,
      "step": 973
    },
    {
      "epoch": 1.27,
      "learning_rate": 3.2820776746434764e-05,
      "logits/chosen": -2.1003689765930176,
      "logits/rejected": -2.096348524093628,
      "logps/chosen": -141.83660888671875,
      "logps/rejected": -200.94760131835938,
      "loss": 0.1068,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0471521615982056,
      "rewards/margins": 4.825916290283203,
      "rewards/rejected": -5.873068332672119,
      "step": 974
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.278673687943011e-05,
      "logits/chosen": -1.9682233333587646,
      "logits/rejected": -1.9423187971115112,
      "logps/chosen": -145.8862762451172,
      "logps/rejected": -204.0359344482422,
      "loss": 0.0704,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2208142280578613,
      "rewards/margins": 5.135477542877197,
      "rewards/rejected": -6.356292724609375,
      "step": 975
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.2752681017800144e-05,
      "logits/chosen": -2.1087942123413086,
      "logits/rejected": -2.081089973449707,
      "logps/chosen": -198.58547973632812,
      "logps/rejected": -238.7448272705078,
      "loss": 0.1334,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2582627534866333,
      "rewards/margins": 5.285311698913574,
      "rewards/rejected": -6.543575286865234,
      "step": 976
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.27186092314985e-05,
      "logits/chosen": -1.8373255729675293,
      "logits/rejected": -1.8873920440673828,
      "logps/chosen": -149.75485229492188,
      "logps/rejected": -203.3557586669922,
      "loss": 0.0728,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4826914966106415,
      "rewards/margins": 6.0817389488220215,
      "rewards/rejected": -6.564430236816406,
      "step": 977
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.2684521590511566e-05,
      "logits/chosen": -1.7711788415908813,
      "logits/rejected": -1.783356785774231,
      "logps/chosen": -167.17437744140625,
      "logps/rejected": -235.35980224609375,
      "loss": 0.0221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3992350399494171,
      "rewards/margins": 7.782591819763184,
      "rewards/rejected": -8.1818265914917,
      "step": 978
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.2650418164858284e-05,
      "logits/chosen": -1.8944889307022095,
      "logits/rejected": -1.8610248565673828,
      "logps/chosen": -171.46087646484375,
      "logps/rejected": -230.45013427734375,
      "loss": 0.0555,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3834775984287262,
      "rewards/margins": 5.588604927062988,
      "rewards/rejected": -5.972082614898682,
      "step": 979
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.261629902459e-05,
      "logits/chosen": -1.941145896911621,
      "logits/rejected": -1.890683889389038,
      "logps/chosen": -162.6816864013672,
      "logps/rejected": -206.3589630126953,
      "loss": 0.0687,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9792903065681458,
      "rewards/margins": 5.750655174255371,
      "rewards/rejected": -6.729945182800293,
      "step": 980
    },
    {
      "epoch": 1.28,
      "learning_rate": 3.258216423979037e-05,
      "logits/chosen": -1.9625548124313354,
      "logits/rejected": -1.982649803161621,
      "logps/chosen": -179.1558380126953,
      "logps/rejected": -259.2472839355469,
      "loss": 0.0974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9090802073478699,
      "rewards/margins": 6.0814690589904785,
      "rewards/rejected": -6.990549564361572,
      "step": 981
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.254801388057514e-05,
      "logits/chosen": -1.9789154529571533,
      "logits/rejected": -2.015522003173828,
      "logps/chosen": -162.9058837890625,
      "logps/rejected": -222.94760131835938,
      "loss": 0.1611,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0825514793395996,
      "rewards/margins": 4.608074188232422,
      "rewards/rejected": -5.690625190734863,
      "step": 982
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.2513848017092113e-05,
      "logits/chosen": -1.927085280418396,
      "logits/rejected": -2.0209970474243164,
      "logps/chosen": -189.93618774414062,
      "logps/rejected": -229.33177185058594,
      "loss": 0.0933,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4452272653579712,
      "rewards/margins": 5.701321601867676,
      "rewards/rejected": -6.146548748016357,
      "step": 983
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.2479666719520886e-05,
      "logits/chosen": -1.8675872087478638,
      "logits/rejected": -1.8806711435317993,
      "logps/chosen": -182.062255859375,
      "logps/rejected": -235.03907775878906,
      "loss": 0.0412,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7320657968521118,
      "rewards/margins": 6.459377288818359,
      "rewards/rejected": -7.191442966461182,
      "step": 984
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.2445470058072766e-05,
      "logits/chosen": -1.8914560079574585,
      "logits/rejected": -1.8393675088882446,
      "logps/chosen": -167.7892608642578,
      "logps/rejected": -226.37176513671875,
      "loss": 0.0132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3088177740573883,
      "rewards/margins": 6.601188659667969,
      "rewards/rejected": -6.910006046295166,
      "step": 985
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.2411258102990646e-05,
      "logits/chosen": -1.9076112508773804,
      "logits/rejected": -1.9284731149673462,
      "logps/chosen": -172.68734741210938,
      "logps/rejected": -241.80279541015625,
      "loss": 0.0473,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.14136190712451935,
      "rewards/margins": 6.534528732299805,
      "rewards/rejected": -6.675890922546387,
      "step": 986
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.23770309245488e-05,
      "logits/chosen": -1.9282286167144775,
      "logits/rejected": -1.8869755268096924,
      "logps/chosen": -185.94140625,
      "logps/rejected": -244.49862670898438,
      "loss": 0.1127,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2529757618904114,
      "rewards/margins": 5.684840679168701,
      "rewards/rejected": -5.937817096710205,
      "step": 987
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.23427885930528e-05,
      "logits/chosen": -1.8192644119262695,
      "logits/rejected": -1.9099791049957275,
      "logps/chosen": -138.94749450683594,
      "logps/rejected": -210.98141479492188,
      "loss": 0.1202,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6651886105537415,
      "rewards/margins": 5.962369441986084,
      "rewards/rejected": -6.62755823135376,
      "step": 988
    },
    {
      "epoch": 1.29,
      "learning_rate": 3.230853117883933e-05,
      "logits/chosen": -1.8780765533447266,
      "logits/rejected": -1.9324958324432373,
      "logps/chosen": -127.44175720214844,
      "logps/rejected": -195.4342041015625,
      "loss": 0.1616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8466498851776123,
      "rewards/margins": 5.705711364746094,
      "rewards/rejected": -6.552361488342285,
      "step": 989
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.227425875227605e-05,
      "logits/chosen": -1.9504503011703491,
      "logits/rejected": -1.9801244735717773,
      "logps/chosen": -151.307861328125,
      "logps/rejected": -232.94631958007812,
      "loss": 0.0913,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4374302327632904,
      "rewards/margins": 5.352200508117676,
      "rewards/rejected": -5.789630889892578,
      "step": 990
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.223997138376146e-05,
      "logits/chosen": -1.768437385559082,
      "logits/rejected": -1.8152680397033691,
      "logps/chosen": -219.896240234375,
      "logps/rejected": -264.9837646484375,
      "loss": 0.1102,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0947672128677368,
      "rewards/margins": 4.561570167541504,
      "rewards/rejected": -5.656337738037109,
      "step": 991
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.220566914372477e-05,
      "logits/chosen": -1.914330244064331,
      "logits/rejected": -1.8707256317138672,
      "logps/chosen": -161.68264770507812,
      "logps/rejected": -206.5924835205078,
      "loss": 0.0468,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0668751001358032,
      "rewards/margins": 4.970590114593506,
      "rewards/rejected": -6.0374650955200195,
      "step": 992
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.2171352102625716e-05,
      "logits/chosen": -1.932759165763855,
      "logits/rejected": -1.9709361791610718,
      "logps/chosen": -215.7425079345703,
      "logps/rejected": -309.05194091796875,
      "loss": 0.0456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2079094648361206,
      "rewards/margins": 7.571928024291992,
      "rewards/rejected": -7.779837608337402,
      "step": 993
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.213702033095444e-05,
      "logits/chosen": -1.8618203401565552,
      "logits/rejected": -1.9024603366851807,
      "logps/chosen": -178.80416870117188,
      "logps/rejected": -260.2957763671875,
      "loss": 0.1162,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.20789635181427,
      "rewards/margins": 6.124366283416748,
      "rewards/rejected": -7.3322625160217285,
      "step": 994
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.210267389923135e-05,
      "logits/chosen": -2.0747745037078857,
      "logits/rejected": -2.092907667160034,
      "logps/chosen": -163.86630249023438,
      "logps/rejected": -208.08639526367188,
      "loss": 0.2317,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4616580009460449,
      "rewards/margins": 4.820524215698242,
      "rewards/rejected": -5.282181739807129,
      "step": 995
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.2068312878006955e-05,
      "logits/chosen": -1.9825048446655273,
      "logits/rejected": -2.042208433151245,
      "logps/chosen": -149.8975372314453,
      "logps/rejected": -213.1818389892578,
      "loss": 0.0461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6773964166641235,
      "rewards/margins": 6.074430465698242,
      "rewards/rejected": -6.751826286315918,
      "step": 996
    },
    {
      "epoch": 1.3,
      "learning_rate": 3.2033937337861744e-05,
      "logits/chosen": -1.456627368927002,
      "logits/rejected": -1.5451298952102661,
      "logps/chosen": -195.9957275390625,
      "logps/rejected": -275.362060546875,
      "loss": 0.0561,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.783312976360321,
      "rewards/margins": 6.08614444732666,
      "rewards/rejected": -6.869457721710205,
      "step": 997
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.199954734940603e-05,
      "logits/chosen": -1.848314881324768,
      "logits/rejected": -1.8288490772247314,
      "logps/chosen": -146.74244689941406,
      "logps/rejected": -214.40316772460938,
      "loss": 0.1002,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5085213780403137,
      "rewards/margins": 6.950279235839844,
      "rewards/rejected": -7.45880126953125,
      "step": 998
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.196514298327979e-05,
      "logits/chosen": -1.9763479232788086,
      "logits/rejected": -1.9941948652267456,
      "logps/chosen": -175.21180725097656,
      "logps/rejected": -211.73851013183594,
      "loss": 0.2338,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7451385259628296,
      "rewards/margins": 4.789652347564697,
      "rewards/rejected": -6.534790992736816,
      "step": 999
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.193072431015254e-05,
      "logits/chosen": -1.9940496683120728,
      "logits/rejected": -1.9827721118927002,
      "logps/chosen": -141.00697326660156,
      "logps/rejected": -209.87010192871094,
      "loss": 0.0899,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5353857278823853,
      "rewards/margins": 6.444552898406982,
      "rewards/rejected": -6.979938507080078,
      "step": 1000
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.18962914007232e-05,
      "logits/chosen": -1.9758892059326172,
      "logits/rejected": -1.989560842514038,
      "logps/chosen": -242.91734313964844,
      "logps/rejected": -266.1889343261719,
      "loss": 0.114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1345280408859253,
      "rewards/margins": 5.276760101318359,
      "rewards/rejected": -6.411287784576416,
      "step": 1001
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.18618443257199e-05,
      "logits/chosen": -2.05208420753479,
      "logits/rejected": -2.0730035305023193,
      "logps/chosen": -247.06094360351562,
      "logps/rejected": -287.7290954589844,
      "loss": 0.0773,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.20817336440086365,
      "rewards/margins": 5.960603713989258,
      "rewards/rejected": -6.168776512145996,
      "step": 1002
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.182738315589991e-05,
      "logits/chosen": -1.760789155960083,
      "logits/rejected": -1.7321034669876099,
      "logps/chosen": -153.32969665527344,
      "logps/rejected": -238.1700439453125,
      "loss": 0.1137,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7017782330513,
      "rewards/margins": 7.256704330444336,
      "rewards/rejected": -7.95848274230957,
      "step": 1003
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.17929079620494e-05,
      "logits/chosen": -1.7065904140472412,
      "logits/rejected": -1.7517046928405762,
      "logps/chosen": -140.71090698242188,
      "logps/rejected": -214.5618896484375,
      "loss": 0.0959,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7964804172515869,
      "rewards/margins": 6.083569049835205,
      "rewards/rejected": -6.880049705505371,
      "step": 1004
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.17584188149834e-05,
      "logits/chosen": -1.9785032272338867,
      "logits/rejected": -1.987661600112915,
      "logps/chosen": -236.944091796875,
      "logps/rejected": -294.55908203125,
      "loss": 0.1802,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8768453598022461,
      "rewards/margins": 5.34891414642334,
      "rewards/rejected": -6.225759506225586,
      "step": 1005
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.172391578554557e-05,
      "logits/chosen": -1.7270407676696777,
      "logits/rejected": -1.8673019409179688,
      "logps/chosen": -167.82101440429688,
      "logps/rejected": -256.2132568359375,
      "loss": 0.022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06484043598175049,
      "rewards/margins": 8.01184368133545,
      "rewards/rejected": -7.947002410888672,
      "step": 1006
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.1689398944608076e-05,
      "logits/chosen": -1.9291620254516602,
      "logits/rejected": -1.935348391532898,
      "logps/chosen": -155.17001342773438,
      "logps/rejected": -262.1614074707031,
      "loss": 0.0097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27539539337158203,
      "rewards/margins": 9.372123718261719,
      "rewards/rejected": -9.09672737121582,
      "step": 1007
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.1654868363071484e-05,
      "logits/chosen": -1.6849298477172852,
      "logits/rejected": -1.7507292032241821,
      "logps/chosen": -224.79669189453125,
      "logps/rejected": -239.8026580810547,
      "loss": 0.2553,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8910402059555054,
      "rewards/margins": 4.9744768142700195,
      "rewards/rejected": -5.8655171394348145,
      "step": 1008
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.162032411186456e-05,
      "logits/chosen": -2.02671217918396,
      "logits/rejected": -2.0000789165496826,
      "logps/chosen": -145.2879638671875,
      "logps/rejected": -207.11729431152344,
      "loss": 0.1097,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8955314755439758,
      "rewards/margins": 6.038568019866943,
      "rewards/rejected": -6.934099197387695,
      "step": 1009
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.158576626194417e-05,
      "logits/chosen": -1.9856791496276855,
      "logits/rejected": -2.0733370780944824,
      "logps/chosen": -166.86439514160156,
      "logps/rejected": -232.08746337890625,
      "loss": 0.081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.135887622833252,
      "rewards/margins": 5.414515018463135,
      "rewards/rejected": -6.550402641296387,
      "step": 1010
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.15511948842951e-05,
      "logits/chosen": -1.9186058044433594,
      "logits/rejected": -1.9732969999313354,
      "logps/chosen": -168.21229553222656,
      "logps/rejected": -230.05770874023438,
      "loss": 0.1944,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1896405220031738,
      "rewards/margins": 6.240389823913574,
      "rewards/rejected": -7.430030345916748,
      "step": 1011
    },
    {
      "epoch": 1.32,
      "learning_rate": 3.151661004992992e-05,
      "logits/chosen": -2.1758668422698975,
      "logits/rejected": -2.2037084102630615,
      "logps/chosen": -227.41525268554688,
      "logps/rejected": -278.093017578125,
      "loss": 0.107,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1396892070770264,
      "rewards/margins": 5.778409957885742,
      "rewards/rejected": -6.9180989265441895,
      "step": 1012
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.1482011829888836e-05,
      "logits/chosen": -2.012368679046631,
      "logits/rejected": -2.033564567565918,
      "logps/chosen": -174.45318603515625,
      "logps/rejected": -222.24986267089844,
      "loss": 0.1195,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6864941120147705,
      "rewards/margins": 5.817201614379883,
      "rewards/rejected": -6.503695964813232,
      "step": 1013
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.1447400295239575e-05,
      "logits/chosen": -1.96452796459198,
      "logits/rejected": -1.9833523035049438,
      "logps/chosen": -173.67625427246094,
      "logps/rejected": -252.71640014648438,
      "loss": 0.1021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9247287511825562,
      "rewards/margins": 7.440773010253906,
      "rewards/rejected": -8.36550235748291,
      "step": 1014
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.1412775517077195e-05,
      "logits/chosen": -1.5514934062957764,
      "logits/rejected": -1.606977105140686,
      "logps/chosen": -129.79220581054688,
      "logps/rejected": -203.38143920898438,
      "loss": 0.2468,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5034909248352051,
      "rewards/margins": 5.886725902557373,
      "rewards/rejected": -6.390216827392578,
      "step": 1015
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.137813756652395e-05,
      "logits/chosen": -1.9654016494750977,
      "logits/rejected": -2.000110626220703,
      "logps/chosen": -156.043701171875,
      "logps/rejected": -215.63888549804688,
      "loss": 0.0285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.019132375717163,
      "rewards/margins": 5.414684295654297,
      "rewards/rejected": -6.433815956115723,
      "step": 1016
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.134348651472917e-05,
      "logits/chosen": -2.0545971393585205,
      "logits/rejected": -2.065199375152588,
      "logps/chosen": -164.96517944335938,
      "logps/rejected": -220.41319274902344,
      "loss": 0.1999,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.487870216369629,
      "rewards/margins": 5.2257280349731445,
      "rewards/rejected": -6.713598251342773,
      "step": 1017
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.130882243286908e-05,
      "logits/chosen": -2.0629384517669678,
      "logits/rejected": -2.1075491905212402,
      "logps/chosen": -167.95260620117188,
      "logps/rejected": -225.93836975097656,
      "loss": 0.2136,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.167354941368103,
      "rewards/margins": 5.635895729064941,
      "rewards/rejected": -6.803251266479492,
      "step": 1018
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.127414539214668e-05,
      "logits/chosen": -1.7879366874694824,
      "logits/rejected": -1.8369427919387817,
      "logps/chosen": -156.32177734375,
      "logps/rejected": -230.6071014404297,
      "loss": 0.0391,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7986168265342712,
      "rewards/margins": 7.002883434295654,
      "rewards/rejected": -7.8015007972717285,
      "step": 1019
    },
    {
      "epoch": 1.33,
      "learning_rate": 3.12394554637916e-05,
      "logits/chosen": -2.068706512451172,
      "logits/rejected": -2.02579665184021,
      "logps/chosen": -159.2906036376953,
      "logps/rejected": -209.74407958984375,
      "loss": 0.0335,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.36703935265541077,
      "rewards/margins": 6.066681861877441,
      "rewards/rejected": -6.43372106552124,
      "step": 1020
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.12047527190599e-05,
      "logits/chosen": -1.9125123023986816,
      "logits/rejected": -1.9503191709518433,
      "logps/chosen": -148.84817504882812,
      "logps/rejected": -221.28292846679688,
      "loss": 0.1086,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6139389276504517,
      "rewards/margins": 7.040698051452637,
      "rewards/rejected": -7.654637336730957,
      "step": 1021
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.1170037229234006e-05,
      "logits/chosen": -1.5111428499221802,
      "logits/rejected": -1.5389604568481445,
      "logps/chosen": -160.67080688476562,
      "logps/rejected": -197.14707946777344,
      "loss": 0.2363,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4887231588363647,
      "rewards/margins": 3.939398765563965,
      "rewards/rejected": -5.428122043609619,
      "step": 1022
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.113530906562252e-05,
      "logits/chosen": -2.0650384426116943,
      "logits/rejected": -1.9774444103240967,
      "logps/chosen": -167.2080841064453,
      "logps/rejected": -226.31394958496094,
      "loss": 0.0251,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4560891389846802,
      "rewards/margins": 7.482360363006592,
      "rewards/rejected": -7.938448905944824,
      "step": 1023
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.110056829956006e-05,
      "logits/chosen": -1.7791208028793335,
      "logits/rejected": -1.7737149000167847,
      "logps/chosen": -157.49781799316406,
      "logps/rejected": -224.4245147705078,
      "loss": 0.0735,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9841396808624268,
      "rewards/margins": 6.30238676071167,
      "rewards/rejected": -7.286526203155518,
      "step": 1024
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.1065815002407136e-05,
      "logits/chosen": -1.7746696472167969,
      "logits/rejected": -1.7677335739135742,
      "logps/chosen": -143.35836791992188,
      "logps/rejected": -197.0070037841797,
      "loss": 0.1212,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3930830657482147,
      "rewards/margins": 6.265653610229492,
      "rewards/rejected": -6.658736705780029,
      "step": 1025
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.103104924555e-05,
      "logits/chosen": -2.0955138206481934,
      "logits/rejected": -2.10188364982605,
      "logps/chosen": -148.90345764160156,
      "logps/rejected": -214.53973388671875,
      "loss": 0.0196,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1263139247894287,
      "rewards/margins": 5.5401692390441895,
      "rewards/rejected": -6.666483402252197,
      "step": 1026
    },
    {
      "epoch": 1.34,
      "learning_rate": 3.099627110040052e-05,
      "logits/chosen": -1.772825837135315,
      "logits/rejected": -1.7642711400985718,
      "logps/chosen": -189.01547241210938,
      "logps/rejected": -231.60894775390625,
      "loss": 0.1176,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8581879734992981,
      "rewards/margins": 6.53199577331543,
      "rewards/rejected": -7.390183448791504,
      "step": 1027
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.096148063839596e-05,
      "logits/chosen": -1.7981985807418823,
      "logits/rejected": -1.8317527770996094,
      "logps/chosen": -221.88800048828125,
      "logps/rejected": -303.67529296875,
      "loss": 0.0889,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.25540804862976074,
      "rewards/margins": 7.973862171173096,
      "rewards/rejected": -8.229269981384277,
      "step": 1028
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0926677930998924e-05,
      "logits/chosen": -2.035282850265503,
      "logits/rejected": -1.9960718154907227,
      "logps/chosen": -146.65322875976562,
      "logps/rejected": -187.8694610595703,
      "loss": 0.1698,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2113975286483765,
      "rewards/margins": 4.118739604949951,
      "rewards/rejected": -5.330136775970459,
      "step": 1029
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0891863049697165e-05,
      "logits/chosen": -1.814779281616211,
      "logits/rejected": -1.8722188472747803,
      "logps/chosen": -193.19418334960938,
      "logps/rejected": -246.48104858398438,
      "loss": 0.133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.67592191696167,
      "rewards/margins": 6.500290393829346,
      "rewards/rejected": -8.176212310791016,
      "step": 1030
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0857036066003414e-05,
      "logits/chosen": -2.0683040618896484,
      "logits/rejected": -2.100090265274048,
      "logps/chosen": -163.79466247558594,
      "logps/rejected": -244.75115966796875,
      "loss": 0.0137,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9199898838996887,
      "rewards/margins": 8.365705490112305,
      "rewards/rejected": -9.28569507598877,
      "step": 1031
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.08221970514553e-05,
      "logits/chosen": -1.8518726825714111,
      "logits/rejected": -1.8920923471450806,
      "logps/chosen": -177.65554809570312,
      "logps/rejected": -229.32919311523438,
      "loss": 0.0414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2180662155151367,
      "rewards/margins": 5.394405364990234,
      "rewards/rejected": -6.612471103668213,
      "step": 1032
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0787346077615155e-05,
      "logits/chosen": -1.8266615867614746,
      "logits/rejected": -1.836155891418457,
      "logps/chosen": -221.51577758789062,
      "logps/rejected": -268.7772521972656,
      "loss": 0.0626,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3641536235809326,
      "rewards/margins": 6.098878383636475,
      "rewards/rejected": -7.463032245635986,
      "step": 1033
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.0752483216069846e-05,
      "logits/chosen": -1.8259131908416748,
      "logits/rejected": -1.7714595794677734,
      "logps/chosen": -193.35919189453125,
      "logps/rejected": -268.484619140625,
      "loss": 0.0574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1007038354873657,
      "rewards/margins": 6.163137435913086,
      "rewards/rejected": -7.26384162902832,
      "step": 1034
    },
    {
      "epoch": 1.35,
      "learning_rate": 3.071760853843069e-05,
      "logits/chosen": -1.6738380193710327,
      "logits/rejected": -1.6765129566192627,
      "logps/chosen": -191.4422607421875,
      "logps/rejected": -229.07388305664062,
      "loss": 0.2031,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.2890628576278687,
      "rewards/margins": 5.058915138244629,
      "rewards/rejected": -6.347978591918945,
      "step": 1035
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.068272211633326e-05,
      "logits/chosen": -1.8329373598098755,
      "logits/rejected": -1.8167811632156372,
      "logps/chosen": -185.8599395751953,
      "logps/rejected": -241.39480590820312,
      "loss": 0.0633,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6821643114089966,
      "rewards/margins": 5.0010905265808105,
      "rewards/rejected": -6.683254241943359,
      "step": 1036
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0647824021437266e-05,
      "logits/chosen": -1.9100103378295898,
      "logits/rejected": -1.8681895732879639,
      "logps/chosen": -173.29039001464844,
      "logps/rejected": -254.75445556640625,
      "loss": 0.0474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2345590591430664,
      "rewards/margins": 6.126624584197998,
      "rewards/rejected": -7.361184120178223,
      "step": 1037
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.061291432542639e-05,
      "logits/chosen": -1.9940087795257568,
      "logits/rejected": -1.906955361366272,
      "logps/chosen": -185.31643676757812,
      "logps/rejected": -255.26145935058594,
      "loss": 0.0494,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.33682963252067566,
      "rewards/margins": 6.860749244689941,
      "rewards/rejected": -7.197578430175781,
      "step": 1038
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0577993100008135e-05,
      "logits/chosen": -1.5305901765823364,
      "logits/rejected": -1.5175468921661377,
      "logps/chosen": -168.8824005126953,
      "logps/rejected": -267.9697265625,
      "loss": 0.0563,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3656330406665802,
      "rewards/margins": 7.50492525100708,
      "rewards/rejected": -7.87055778503418,
      "step": 1039
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0543060416913696e-05,
      "logits/chosen": -1.607844352722168,
      "logits/rejected": -1.6281657218933105,
      "logps/chosen": -155.04019165039062,
      "logps/rejected": -225.8870391845703,
      "loss": 0.0371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7197632789611816,
      "rewards/margins": 6.631788730621338,
      "rewards/rejected": -7.3515520095825195,
      "step": 1040
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.050811634789779e-05,
      "logits/chosen": -1.7357194423675537,
      "logits/rejected": -1.795137882232666,
      "logps/chosen": -152.45477294921875,
      "logps/rejected": -225.6587371826172,
      "loss": 0.1147,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0607367753982544,
      "rewards/margins": 6.239053726196289,
      "rewards/rejected": -7.299789905548096,
      "step": 1041
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0473160964738555e-05,
      "logits/chosen": -1.5338330268859863,
      "logits/rejected": -1.563486099243164,
      "logps/chosen": -159.23626708984375,
      "logps/rejected": -223.11581420898438,
      "loss": 0.1031,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8340301513671875,
      "rewards/margins": 6.626452445983887,
      "rewards/rejected": -7.460483074188232,
      "step": 1042
    },
    {
      "epoch": 1.36,
      "learning_rate": 3.0438194339237325e-05,
      "logits/chosen": -1.8667819499969482,
      "logits/rejected": -1.8537089824676514,
      "logps/chosen": -192.2030487060547,
      "logps/rejected": -229.79701232910156,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1402863264083862,
      "rewards/margins": 5.89219856262207,
      "rewards/rejected": -7.032485008239746,
      "step": 1043
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.0403216543218547e-05,
      "logits/chosen": -1.9166359901428223,
      "logits/rejected": -1.9331247806549072,
      "logps/chosen": -157.19808959960938,
      "logps/rejected": -227.13455200195312,
      "loss": 0.0173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.1606651246547699,
      "rewards/margins": 6.456012725830078,
      "rewards/rejected": -6.616677761077881,
      "step": 1044
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.036822764852963e-05,
      "logits/chosen": -1.8743844032287598,
      "logits/rejected": -1.912811040878296,
      "logps/chosen": -170.45713806152344,
      "logps/rejected": -247.20166015625,
      "loss": 0.0362,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40381884574890137,
      "rewards/margins": 6.9431471824646,
      "rewards/rejected": -7.346966743469238,
      "step": 1045
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.0333227727040742e-05,
      "logits/chosen": -1.84022057056427,
      "logits/rejected": -1.8556888103485107,
      "logps/chosen": -137.298828125,
      "logps/rejected": -193.88853454589844,
      "loss": 0.1473,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0216233730316162,
      "rewards/margins": 4.4968342781066895,
      "rewards/rejected": -5.518457889556885,
      "step": 1046
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.029821685064475e-05,
      "logits/chosen": -1.8517441749572754,
      "logits/rejected": -2.004195213317871,
      "logps/chosen": -157.5634765625,
      "logps/rejected": -270.2324523925781,
      "loss": 0.0139,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.545246958732605,
      "rewards/margins": 8.651532173156738,
      "rewards/rejected": -9.196779251098633,
      "step": 1047
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.026319509125697e-05,
      "logits/chosen": -1.7293380498886108,
      "logits/rejected": -1.8183757066726685,
      "logps/chosen": -167.41976928710938,
      "logps/rejected": -219.95889282226562,
      "loss": 0.1151,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.45486682653427124,
      "rewards/margins": 5.838650703430176,
      "rewards/rejected": -6.293517589569092,
      "step": 1048
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.0228162520815117e-05,
      "logits/chosen": -1.8373677730560303,
      "logits/rejected": -1.806967854499817,
      "logps/chosen": -146.26051330566406,
      "logps/rejected": -205.7896728515625,
      "loss": 0.0709,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6990506649017334,
      "rewards/margins": 6.332763195037842,
      "rewards/rejected": -7.0318145751953125,
      "step": 1049
    },
    {
      "epoch": 1.37,
      "learning_rate": 3.0193119211279097e-05,
      "logits/chosen": -1.5211528539657593,
      "logits/rejected": -1.4527685642242432,
      "logps/chosen": -164.65892028808594,
      "logps/rejected": -246.6031036376953,
      "loss": 0.0729,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2956247329711914,
      "rewards/margins": 5.807753562927246,
      "rewards/rejected": -7.103377819061279,
      "step": 1050
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.015806523463085e-05,
      "logits/chosen": -1.8467833995819092,
      "logits/rejected": -1.8871638774871826,
      "logps/chosen": -164.1189727783203,
      "logps/rejected": -231.1582489013672,
      "loss": 0.0045,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14765971899032593,
      "rewards/margins": 7.357109069824219,
      "rewards/rejected": -7.20944881439209,
      "step": 1051
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.0123000662874272e-05,
      "logits/chosen": -1.8304824829101562,
      "logits/rejected": -1.8351521492004395,
      "logps/chosen": -217.35919189453125,
      "logps/rejected": -255.71145629882812,
      "loss": 0.2171,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6964375972747803,
      "rewards/margins": 4.231793403625488,
      "rewards/rejected": -5.928231239318848,
      "step": 1052
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.0087925568034998e-05,
      "logits/chosen": -1.9726256132125854,
      "logits/rejected": -1.9677804708480835,
      "logps/chosen": -152.54608154296875,
      "logps/rejected": -223.62603759765625,
      "loss": 0.0104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22661477327346802,
      "rewards/margins": 7.715180397033691,
      "rewards/rejected": -7.941795349121094,
      "step": 1053
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.0052840022160273e-05,
      "logits/chosen": -1.8187848329544067,
      "logits/rejected": -1.8540804386138916,
      "logps/chosen": -179.23916625976562,
      "logps/rejected": -239.16238403320312,
      "loss": 0.0981,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2303731441497803,
      "rewards/margins": 5.892210960388184,
      "rewards/rejected": -7.122584342956543,
      "step": 1054
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.0017744097318823e-05,
      "logits/chosen": -2.000126361846924,
      "logits/rejected": -2.0695457458496094,
      "logps/chosen": -159.04696655273438,
      "logps/rejected": -239.05096435546875,
      "loss": 0.1061,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8145641684532166,
      "rewards/margins": 6.3894195556640625,
      "rewards/rejected": -7.203983783721924,
      "step": 1055
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.9982637865600683e-05,
      "logits/chosen": -1.9221899509429932,
      "logits/rejected": -2.00850248336792,
      "logps/chosen": -145.6482696533203,
      "logps/rejected": -224.4560546875,
      "loss": 0.0705,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0264939069747925,
      "rewards/margins": 6.882168292999268,
      "rewards/rejected": -7.908662796020508,
      "step": 1056
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.994752139911706e-05,
      "logits/chosen": -1.7955631017684937,
      "logits/rejected": -1.823647379875183,
      "logps/chosen": -176.24493408203125,
      "logps/rejected": -255.06854248046875,
      "loss": 0.0425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8654636144638062,
      "rewards/margins": 7.487548828125,
      "rewards/rejected": -8.353012084960938,
      "step": 1057
    },
    {
      "epoch": 1.38,
      "learning_rate": 2.991239477000021e-05,
      "logits/chosen": -1.8705966472625732,
      "logits/rejected": -1.8845902681350708,
      "logps/chosen": -164.41986083984375,
      "logps/rejected": -206.58514404296875,
      "loss": 0.1374,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.151316523551941,
      "rewards/margins": 5.059691905975342,
      "rewards/rejected": -6.211008071899414,
      "step": 1058
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9877258050403212e-05,
      "logits/chosen": -1.5746783018112183,
      "logits/rejected": -1.6217272281646729,
      "logps/chosen": -149.62371826171875,
      "logps/rejected": -244.63250732421875,
      "loss": 0.0199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7274661064147949,
      "rewards/margins": 7.111369609832764,
      "rewards/rejected": -7.838835716247559,
      "step": 1059
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9842111312499914e-05,
      "logits/chosen": -1.9484206438064575,
      "logits/rejected": -1.9830492734909058,
      "logps/chosen": -158.52488708496094,
      "logps/rejected": -220.32965087890625,
      "loss": 0.1044,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.42652958631515503,
      "rewards/margins": 6.116347789764404,
      "rewards/rejected": -6.542877197265625,
      "step": 1060
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9806954628484734e-05,
      "logits/chosen": -1.8739213943481445,
      "logits/rejected": -1.878121018409729,
      "logps/chosen": -140.96554565429688,
      "logps/rejected": -201.78395080566406,
      "loss": 0.0408,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5415512323379517,
      "rewards/margins": 5.48455810546875,
      "rewards/rejected": -6.02610969543457,
      "step": 1061
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9771788070572514e-05,
      "logits/chosen": -1.8523356914520264,
      "logits/rejected": -1.8917300701141357,
      "logps/chosen": -150.42018127441406,
      "logps/rejected": -241.1668243408203,
      "loss": 0.0872,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6819233894348145,
      "rewards/margins": 7.713897705078125,
      "rewards/rejected": -8.395821571350098,
      "step": 1062
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9736611710998368e-05,
      "logits/chosen": -1.8981248140335083,
      "logits/rejected": -1.980208158493042,
      "logps/chosen": -169.017578125,
      "logps/rejected": -246.5729217529297,
      "loss": 0.1213,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6533073782920837,
      "rewards/margins": 7.117403984069824,
      "rewards/rejected": -7.770711898803711,
      "step": 1063
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9701425622017583e-05,
      "logits/chosen": -1.8987089395523071,
      "logits/rejected": -1.9092296361923218,
      "logps/chosen": -164.01492309570312,
      "logps/rejected": -245.9884490966797,
      "loss": 0.0674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7150200009346008,
      "rewards/margins": 6.637688636779785,
      "rewards/rejected": -7.352709770202637,
      "step": 1064
    },
    {
      "epoch": 1.39,
      "learning_rate": 2.9666229875905373e-05,
      "logits/chosen": -1.9255547523498535,
      "logits/rejected": -2.000413417816162,
      "logps/chosen": -150.02792358398438,
      "logps/rejected": -255.81008911132812,
      "loss": 0.0315,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16459935903549194,
      "rewards/margins": 8.996576309204102,
      "rewards/rejected": -8.831976890563965,
      "step": 1065
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.963102454495683e-05,
      "logits/chosen": -1.7589186429977417,
      "logits/rejected": -1.7570122480392456,
      "logps/chosen": -150.57115173339844,
      "logps/rejected": -247.15155029296875,
      "loss": 0.038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3479610085487366,
      "rewards/margins": 7.818378448486328,
      "rewards/rejected": -8.166338920593262,
      "step": 1066
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.959580970148673e-05,
      "logits/chosen": -1.9616655111312866,
      "logits/rejected": -1.9504739046096802,
      "logps/chosen": -179.95309448242188,
      "logps/rejected": -255.13372802734375,
      "loss": 0.0203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5499464869499207,
      "rewards/margins": 7.185676097869873,
      "rewards/rejected": -7.735622406005859,
      "step": 1067
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9560585417829368e-05,
      "logits/chosen": -1.6697150468826294,
      "logits/rejected": -1.6883552074432373,
      "logps/chosen": -155.16110229492188,
      "logps/rejected": -212.0230255126953,
      "loss": 0.0541,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9748433828353882,
      "rewards/margins": 5.495103359222412,
      "rewards/rejected": -6.469947338104248,
      "step": 1068
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.952535176633846e-05,
      "logits/chosen": -1.68886399269104,
      "logits/rejected": -1.6953492164611816,
      "logps/chosen": -149.33474731445312,
      "logps/rejected": -234.13870239257812,
      "loss": 0.057,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7257979512214661,
      "rewards/margins": 6.896897315979004,
      "rewards/rejected": -7.622694492340088,
      "step": 1069
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9490108819386936e-05,
      "logits/chosen": -1.955090880393982,
      "logits/rejected": -2.02659273147583,
      "logps/chosen": -164.72958374023438,
      "logps/rejected": -239.1529541015625,
      "loss": 0.0998,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5435435771942139,
      "rewards/margins": 6.066895008087158,
      "rewards/rejected": -7.610438346862793,
      "step": 1070
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.945485664936683e-05,
      "logits/chosen": -1.9174976348876953,
      "logits/rejected": -1.934536099433899,
      "logps/chosen": -169.5382080078125,
      "logps/rejected": -229.3955535888672,
      "loss": 0.0174,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1310392618179321,
      "rewards/margins": 7.241247177124023,
      "rewards/rejected": -8.372285842895508,
      "step": 1071
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.9419595328689138e-05,
      "logits/chosen": -1.9342780113220215,
      "logits/rejected": -1.9647936820983887,
      "logps/chosen": -156.1199951171875,
      "logps/rejected": -230.37522888183594,
      "loss": 0.1428,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0096255540847778,
      "rewards/margins": 5.9147562980651855,
      "rewards/rejected": -6.924382209777832,
      "step": 1072
    },
    {
      "epoch": 1.4,
      "learning_rate": 2.938432492978361e-05,
      "logits/chosen": -1.7691768407821655,
      "logits/rejected": -1.7550435066223145,
      "logps/chosen": -195.5511016845703,
      "logps/rejected": -258.09954833984375,
      "loss": 0.1939,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0528061389923096,
      "rewards/margins": 5.208138465881348,
      "rewards/rejected": -6.2609453201293945,
      "step": 1073
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9349045525098688e-05,
      "logits/chosen": -1.9772098064422607,
      "logits/rejected": -2.0032455921173096,
      "logps/chosen": -132.9622344970703,
      "logps/rejected": -229.1063995361328,
      "loss": 0.0461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3998850882053375,
      "rewards/margins": 8.24769401550293,
      "rewards/rejected": -8.647579193115234,
      "step": 1074
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9313757187101297e-05,
      "logits/chosen": -2.0155134201049805,
      "logits/rejected": -1.9626569747924805,
      "logps/chosen": -148.441650390625,
      "logps/rejected": -190.07363891601562,
      "loss": 0.0712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.21381643414497375,
      "rewards/margins": 6.214961051940918,
      "rewards/rejected": -6.428777694702148,
      "step": 1075
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9278459988276703e-05,
      "logits/chosen": -1.8363666534423828,
      "logits/rejected": -1.8916099071502686,
      "logps/chosen": -158.68670654296875,
      "logps/rejected": -208.44607543945312,
      "loss": 0.0586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3279597759246826,
      "rewards/margins": 5.506262302398682,
      "rewards/rejected": -6.834222316741943,
      "step": 1076
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9243154001128386e-05,
      "logits/chosen": -1.8487452268600464,
      "logits/rejected": -1.8077938556671143,
      "logps/chosen": -150.52655029296875,
      "logps/rejected": -211.71670532226562,
      "loss": 0.0326,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22937673330307007,
      "rewards/margins": 7.442183971405029,
      "rewards/rejected": -7.212807655334473,
      "step": 1077
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.920783929817786e-05,
      "logits/chosen": -1.9180998802185059,
      "logits/rejected": -1.9118893146514893,
      "logps/chosen": -159.90924072265625,
      "logps/rejected": -211.256103515625,
      "loss": 0.0894,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.050107002258301,
      "rewards/margins": 5.346580505371094,
      "rewards/rejected": -7.3966875076293945,
      "step": 1078
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9172515951964558e-05,
      "logits/chosen": -1.781477451324463,
      "logits/rejected": -1.8339203596115112,
      "logps/chosen": -147.5101318359375,
      "logps/rejected": -204.47125244140625,
      "loss": 0.1729,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1276333332061768,
      "rewards/margins": 6.145793914794922,
      "rewards/rejected": -7.273427963256836,
      "step": 1079
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.913718403504567e-05,
      "logits/chosen": -1.8072116374969482,
      "logits/rejected": -1.885046362876892,
      "logps/chosen": -149.96539306640625,
      "logps/rejected": -231.3168182373047,
      "loss": 0.0681,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2060569524765015,
      "rewards/margins": 7.473471164703369,
      "rewards/rejected": -8.67952823638916,
      "step": 1080
    },
    {
      "epoch": 1.41,
      "learning_rate": 2.9101843619995968e-05,
      "logits/chosen": -1.8434984683990479,
      "logits/rejected": -1.896488070487976,
      "logps/chosen": -189.40322875976562,
      "logps/rejected": -256.78851318359375,
      "loss": 0.0525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.41508758068084717,
      "rewards/margins": 6.4853997230529785,
      "rewards/rejected": -6.900486946105957,
      "step": 1081
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.906649477940771e-05,
      "logits/chosen": -1.8969818353652954,
      "logits/rejected": -1.9368399381637573,
      "logps/chosen": -160.4936065673828,
      "logps/rejected": -226.63735961914062,
      "loss": 0.0604,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8661583065986633,
      "rewards/margins": 5.8458452224731445,
      "rewards/rejected": -6.7120041847229,
      "step": 1082
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.9031137585890445e-05,
      "logits/chosen": -2.048954486846924,
      "logits/rejected": -1.9439265727996826,
      "logps/chosen": -168.93434143066406,
      "logps/rejected": -204.03244018554688,
      "loss": 0.0111,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5086731910705566,
      "rewards/margins": 5.967591285705566,
      "rewards/rejected": -6.476263999938965,
      "step": 1083
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.899577211207087e-05,
      "logits/chosen": -1.560281753540039,
      "logits/rejected": -1.575604796409607,
      "logps/chosen": -204.13381958007812,
      "logps/rejected": -274.3894958496094,
      "loss": 0.0969,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18771465122699738,
      "rewards/margins": 7.673420429229736,
      "rewards/rejected": -7.861135959625244,
      "step": 1084
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.89603984305927e-05,
      "logits/chosen": -1.790480613708496,
      "logits/rejected": -1.8134186267852783,
      "logps/chosen": -161.66775512695312,
      "logps/rejected": -239.03912353515625,
      "loss": 0.0693,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03652607649564743,
      "rewards/margins": 8.907902717590332,
      "rewards/rejected": -8.871376037597656,
      "step": 1085
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.8925016614116534e-05,
      "logits/chosen": -1.423715591430664,
      "logits/rejected": -1.373763918876648,
      "logps/chosen": -185.72535705566406,
      "logps/rejected": -262.8277893066406,
      "loss": 0.0874,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2692791819572449,
      "rewards/margins": 7.3683366775512695,
      "rewards/rejected": -7.637616157531738,
      "step": 1086
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.8889626735319635e-05,
      "logits/chosen": -1.564412236213684,
      "logits/rejected": -1.6429688930511475,
      "logps/chosen": -178.7507781982422,
      "logps/rejected": -242.05615234375,
      "loss": 0.16,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6229792237281799,
      "rewards/margins": 6.049881935119629,
      "rewards/rejected": -6.672861099243164,
      "step": 1087
    },
    {
      "epoch": 1.42,
      "learning_rate": 2.8854228866895855e-05,
      "logits/chosen": -1.8641536235809326,
      "logits/rejected": -1.8356475830078125,
      "logps/chosen": -146.39022827148438,
      "logps/rejected": -186.7581787109375,
      "loss": 0.1285,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.19931884109973907,
      "rewards/margins": 5.500951766967773,
      "rewards/rejected": -5.700271129608154,
      "step": 1088
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.8818823081555445e-05,
      "logits/chosen": -1.7928270101547241,
      "logits/rejected": -1.8359030485153198,
      "logps/chosen": -177.76258850097656,
      "logps/rejected": -242.41912841796875,
      "loss": 0.107,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.15263700485229492,
      "rewards/margins": 6.399103164672852,
      "rewards/rejected": -6.551739692687988,
      "step": 1089
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.8783409452024934e-05,
      "logits/chosen": -1.9091498851776123,
      "logits/rejected": -1.997072458267212,
      "logps/chosen": -167.573974609375,
      "logps/rejected": -239.23220825195312,
      "loss": 0.0365,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4648070335388184,
      "rewards/margins": 6.935224533081055,
      "rewards/rejected": -8.400030136108398,
      "step": 1090
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.874798805104696e-05,
      "logits/chosen": -1.9259464740753174,
      "logits/rejected": -1.8891761302947998,
      "logps/chosen": -185.05564880371094,
      "logps/rejected": -237.5453643798828,
      "loss": 0.0894,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6918740272521973,
      "rewards/margins": 4.976701736450195,
      "rewards/rejected": -6.668575763702393,
      "step": 1091
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.8712558951380097e-05,
      "logits/chosen": -1.776898980140686,
      "logits/rejected": -1.7845293283462524,
      "logps/chosen": -160.90005493164062,
      "logps/rejected": -248.22189331054688,
      "loss": 0.0918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9049853086471558,
      "rewards/margins": 7.228081703186035,
      "rewards/rejected": -8.13306713104248,
      "step": 1092
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.867712222579877e-05,
      "logits/chosen": -2.0132358074188232,
      "logits/rejected": -1.8840793371200562,
      "logps/chosen": -219.48904418945312,
      "logps/rejected": -274.2969970703125,
      "loss": 0.0731,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6770806908607483,
      "rewards/margins": 6.275861740112305,
      "rewards/rejected": -6.952942371368408,
      "step": 1093
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.864167794709305e-05,
      "logits/chosen": -1.7688848972320557,
      "logits/rejected": -1.8601479530334473,
      "logps/chosen": -161.01063537597656,
      "logps/rejected": -231.13931274414062,
      "loss": 0.048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.864311158657074,
      "rewards/margins": 6.719928741455078,
      "rewards/rejected": -7.584239959716797,
      "step": 1094
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.860622618806852e-05,
      "logits/chosen": -1.996049404144287,
      "logits/rejected": -2.010871410369873,
      "logps/chosen": -189.35972595214844,
      "logps/rejected": -234.64430236816406,
      "loss": 0.1592,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1617563962936401,
      "rewards/margins": 5.092099666595459,
      "rewards/rejected": -6.2538557052612305,
      "step": 1095
    },
    {
      "epoch": 1.43,
      "learning_rate": 2.857076702154614e-05,
      "logits/chosen": -1.7903379201889038,
      "logits/rejected": -1.8177404403686523,
      "logps/chosen": -196.01995849609375,
      "logps/rejected": -269.8954772949219,
      "loss": 0.1174,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.075880765914917,
      "rewards/margins": 5.788846015930176,
      "rewards/rejected": -6.8647260665893555,
      "step": 1096
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8535300520362075e-05,
      "logits/chosen": -1.9214318990707397,
      "logits/rejected": -1.9189565181732178,
      "logps/chosen": -157.75491333007812,
      "logps/rejected": -212.49713134765625,
      "loss": 0.066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.533260703086853,
      "rewards/margins": 5.300146579742432,
      "rewards/rejected": -6.833407402038574,
      "step": 1097
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.849982675736756e-05,
      "logits/chosen": -1.813687801361084,
      "logits/rejected": -1.8973989486694336,
      "logps/chosen": -164.34608459472656,
      "logps/rejected": -214.61558532714844,
      "loss": 0.0817,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.16184663772583,
      "rewards/margins": 5.029347896575928,
      "rewards/rejected": -6.1911940574646,
      "step": 1098
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8464345805428753e-05,
      "logits/chosen": -1.8911590576171875,
      "logits/rejected": -1.9257155656814575,
      "logps/chosen": -198.2843475341797,
      "logps/rejected": -273.1786193847656,
      "loss": 0.1058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6122144460678101,
      "rewards/margins": 6.894351005554199,
      "rewards/rejected": -7.506566047668457,
      "step": 1099
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8428857737426556e-05,
      "logits/chosen": -1.9251108169555664,
      "logits/rejected": -1.922592282295227,
      "logps/chosen": -188.81126403808594,
      "logps/rejected": -240.57691955566406,
      "loss": 0.0721,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9741132259368896,
      "rewards/margins": 6.1875739097595215,
      "rewards/rejected": -7.16168737411499,
      "step": 1100
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.839336262625652e-05,
      "logits/chosen": -1.8882629871368408,
      "logits/rejected": -1.8738603591918945,
      "logps/chosen": -178.04522705078125,
      "logps/rejected": -232.04568481445312,
      "loss": 0.1216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.095766305923462,
      "rewards/margins": 4.878605842590332,
      "rewards/rejected": -6.974372386932373,
      "step": 1101
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.835786054482864e-05,
      "logits/chosen": -1.7042306661605835,
      "logits/rejected": -1.7519516944885254,
      "logps/chosen": -157.7931671142578,
      "logps/rejected": -242.39117431640625,
      "loss": 0.1076,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5546727180480957,
      "rewards/margins": 6.193284034729004,
      "rewards/rejected": -7.747956275939941,
      "step": 1102
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.832235156606724e-05,
      "logits/chosen": -1.5487494468688965,
      "logits/rejected": -1.559239387512207,
      "logps/chosen": -166.17710876464844,
      "logps/rejected": -250.43637084960938,
      "loss": 0.0457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0496666431427002,
      "rewards/margins": 7.9584174156188965,
      "rewards/rejected": -9.00808334350586,
      "step": 1103
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8286835762910803e-05,
      "logits/chosen": -1.8385528326034546,
      "logits/rejected": -1.820473313331604,
      "logps/chosen": -162.75796508789062,
      "logps/rejected": -204.71331787109375,
      "loss": 0.1341,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2836222648620605,
      "rewards/margins": 5.282740592956543,
      "rewards/rejected": -6.566362380981445,
      "step": 1104
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8251313208311837e-05,
      "logits/chosen": -1.7272975444793701,
      "logits/rejected": -1.7614984512329102,
      "logps/chosen": -163.27529907226562,
      "logps/rejected": -250.61497497558594,
      "loss": 0.0292,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7623785734176636,
      "rewards/margins": 8.864646911621094,
      "rewards/rejected": -9.62702465057373,
      "step": 1105
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8215783975236715e-05,
      "logits/chosen": -1.8095554113388062,
      "logits/rejected": -1.72380530834198,
      "logps/chosen": -161.0970001220703,
      "logps/rejected": -223.05044555664062,
      "loss": 0.0173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25540074706077576,
      "rewards/margins": 7.981770992279053,
      "rewards/rejected": -7.726369857788086,
      "step": 1106
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8180248136665527e-05,
      "logits/chosen": -1.8653849363327026,
      "logits/rejected": -1.8740029335021973,
      "logps/chosen": -172.0203094482422,
      "logps/rejected": -261.2825927734375,
      "loss": 0.0135,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.30215299129486084,
      "rewards/margins": 9.179569244384766,
      "rewards/rejected": -9.481721878051758,
      "step": 1107
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.8144705765591938e-05,
      "logits/chosen": -1.8603312969207764,
      "logits/rejected": -1.8506742715835571,
      "logps/chosen": -164.89642333984375,
      "logps/rejected": -211.65536499023438,
      "loss": 0.0805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2426362037658691,
      "rewards/margins": 5.301024436950684,
      "rewards/rejected": -6.543660640716553,
      "step": 1108
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.810915693502302e-05,
      "logits/chosen": -1.7446722984313965,
      "logits/rejected": -1.7962480783462524,
      "logps/chosen": -226.8018341064453,
      "logps/rejected": -288.31585693359375,
      "loss": 0.1034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9364006519317627,
      "rewards/margins": 6.685792446136475,
      "rewards/rejected": -7.622193336486816,
      "step": 1109
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.807360171797912e-05,
      "logits/chosen": -2.009614944458008,
      "logits/rejected": -1.981621265411377,
      "logps/chosen": -200.21102905273438,
      "logps/rejected": -267.802001953125,
      "loss": 0.2062,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.760534405708313,
      "rewards/margins": 7.430848121643066,
      "rewards/rejected": -9.191383361816406,
      "step": 1110
    },
    {
      "epoch": 1.45,
      "learning_rate": 2.803804018749371e-05,
      "logits/chosen": -1.8083683252334595,
      "logits/rejected": -1.8526943922042847,
      "logps/chosen": -151.57113647460938,
      "logps/rejected": -217.16873168945312,
      "loss": 0.0805,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6117242574691772,
      "rewards/margins": 5.261826992034912,
      "rewards/rejected": -6.873551368713379,
      "step": 1111
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.800247241661321e-05,
      "logits/chosen": -1.5185033082962036,
      "logits/rejected": -1.6484371423721313,
      "logps/chosen": -124.70309448242188,
      "logps/rejected": -185.09490966796875,
      "loss": 0.1213,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0290828943252563,
      "rewards/margins": 5.052825927734375,
      "rewards/rejected": -6.081908226013184,
      "step": 1112
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.796689847839689e-05,
      "logits/chosen": -1.4334614276885986,
      "logits/rejected": -1.5079349279403687,
      "logps/chosen": -150.09938049316406,
      "logps/rejected": -219.8390350341797,
      "loss": 0.1619,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4400235414505005,
      "rewards/margins": 5.5373125076293945,
      "rewards/rejected": -6.977335453033447,
      "step": 1113
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.793131844591666e-05,
      "logits/chosen": -1.7167236804962158,
      "logits/rejected": -1.739565134048462,
      "logps/chosen": -222.3330535888672,
      "logps/rejected": -293.5643615722656,
      "loss": 0.1004,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -3.0473883152008057,
      "rewards/margins": 6.619976997375488,
      "rewards/rejected": -9.667365074157715,
      "step": 1114
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.7895732392256952e-05,
      "logits/chosen": -1.8839598894119263,
      "logits/rejected": -1.834544062614441,
      "logps/chosen": -202.12384033203125,
      "logps/rejected": -245.08433532714844,
      "loss": 0.1314,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.494832158088684,
      "rewards/margins": 5.770140647888184,
      "rewards/rejected": -7.264972686767578,
      "step": 1115
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.7860140390514583e-05,
      "logits/chosen": -1.546716570854187,
      "logits/rejected": -1.5257359743118286,
      "logps/chosen": -167.00848388671875,
      "logps/rejected": -291.791015625,
      "loss": 0.0426,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6200900077819824,
      "rewards/margins": 8.65611457824707,
      "rewards/rejected": -9.276205062866211,
      "step": 1116
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.7824542513798567e-05,
      "logits/chosen": -1.8405026197433472,
      "logits/rejected": -1.9093915224075317,
      "logps/chosen": -168.88865661621094,
      "logps/rejected": -234.0762939453125,
      "loss": 0.0813,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1057682037353516,
      "rewards/margins": 6.423710823059082,
      "rewards/rejected": -7.529479026794434,
      "step": 1117
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.7788938835230005e-05,
      "logits/chosen": -1.8437451124191284,
      "logits/rejected": -1.911022663116455,
      "logps/chosen": -172.62774658203125,
      "logps/rejected": -231.57009887695312,
      "loss": 0.1068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.676173448562622,
      "rewards/margins": 6.035118103027344,
      "rewards/rejected": -7.711291313171387,
      "step": 1118
    },
    {
      "epoch": 1.46,
      "learning_rate": 2.77533294279419e-05,
      "logits/chosen": -1.4455645084381104,
      "logits/rejected": -1.4626951217651367,
      "logps/chosen": -162.79165649414062,
      "logps/rejected": -243.7604522705078,
      "loss": 0.0884,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2320985794067383,
      "rewards/margins": 6.729938983917236,
      "rewards/rejected": -7.962037563323975,
      "step": 1119
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.771771436507903e-05,
      "logits/chosen": -1.871506929397583,
      "logits/rejected": -1.9360620975494385,
      "logps/chosen": -146.05889892578125,
      "logps/rejected": -227.86212158203125,
      "loss": 0.1698,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5294501781463623,
      "rewards/margins": 7.270684242248535,
      "rewards/rejected": -7.800134181976318,
      "step": 1120
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.7682093719797792e-05,
      "logits/chosen": -1.440305233001709,
      "logits/rejected": -1.4973654747009277,
      "logps/chosen": -160.90452575683594,
      "logps/rejected": -237.6295166015625,
      "loss": 0.059,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.427338182926178,
      "rewards/margins": 6.843066692352295,
      "rewards/rejected": -7.270404815673828,
      "step": 1121
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.764646756526603e-05,
      "logits/chosen": -1.9717175960540771,
      "logits/rejected": -1.9232532978057861,
      "logps/chosen": -176.71255493164062,
      "logps/rejected": -233.62005615234375,
      "loss": 0.0521,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3628699779510498,
      "rewards/margins": 7.127203464508057,
      "rewards/rejected": -8.490074157714844,
      "step": 1122
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.7610835974662942e-05,
      "logits/chosen": -1.9445685148239136,
      "logits/rejected": -1.980481743812561,
      "logps/chosen": -165.49476623535156,
      "logps/rejected": -211.83811950683594,
      "loss": 0.1686,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1542731523513794,
      "rewards/margins": 5.7306342124938965,
      "rewards/rejected": -6.884907245635986,
      "step": 1123
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.757519902117886e-05,
      "logits/chosen": -1.850689172744751,
      "logits/rejected": -1.9348920583724976,
      "logps/chosen": -171.9382781982422,
      "logps/rejected": -261.599365234375,
      "loss": 0.0062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8366376757621765,
      "rewards/margins": 8.086565971374512,
      "rewards/rejected": -8.923202514648438,
      "step": 1124
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.7539556778015147e-05,
      "logits/chosen": -1.5381782054901123,
      "logits/rejected": -1.467329740524292,
      "logps/chosen": -180.92552185058594,
      "logps/rejected": -242.7810516357422,
      "loss": 0.1617,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0685694217681885,
      "rewards/margins": 5.840105056762695,
      "rewards/rejected": -7.908675193786621,
      "step": 1125
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.7503909318384026e-05,
      "logits/chosen": -1.782409906387329,
      "logits/rejected": -1.7943681478500366,
      "logps/chosen": -154.49868774414062,
      "logps/rejected": -211.05255126953125,
      "loss": 0.0405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2807073593139648,
      "rewards/margins": 5.9702301025390625,
      "rewards/rejected": -7.250937461853027,
      "step": 1126
    },
    {
      "epoch": 1.47,
      "learning_rate": 2.7468256715508428e-05,
      "logits/chosen": -1.9016053676605225,
      "logits/rejected": -1.8824925422668457,
      "logps/chosen": -144.10421752929688,
      "logps/rejected": -219.752685546875,
      "loss": 0.0537,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5313291549682617,
      "rewards/margins": 6.343008041381836,
      "rewards/rejected": -7.874337196350098,
      "step": 1127
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.743259904262187e-05,
      "logits/chosen": -1.8549048900604248,
      "logits/rejected": -1.907701015472412,
      "logps/chosen": -179.58206176757812,
      "logps/rejected": -239.962890625,
      "loss": 0.1276,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0663328170776367,
      "rewards/margins": 6.683225631713867,
      "rewards/rejected": -8.749558448791504,
      "step": 1128
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.739693637296826e-05,
      "logits/chosen": -1.898420810699463,
      "logits/rejected": -1.9032244682312012,
      "logps/chosen": -234.26800537109375,
      "logps/rejected": -295.5345153808594,
      "loss": 0.1536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5871583223342896,
      "rewards/margins": 5.631552219390869,
      "rewards/rejected": -7.218710422515869,
      "step": 1129
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.7361268779801785e-05,
      "logits/chosen": -1.9281286001205444,
      "logits/rejected": -1.9318814277648926,
      "logps/chosen": -150.0965576171875,
      "logps/rejected": -189.6211395263672,
      "loss": 0.1412,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9154345989227295,
      "rewards/margins": 4.384958267211914,
      "rewards/rejected": -6.300393104553223,
      "step": 1130
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.7325596336386738e-05,
      "logits/chosen": -1.7077229022979736,
      "logits/rejected": -1.7676060199737549,
      "logps/chosen": -170.3775634765625,
      "logps/rejected": -217.70330810546875,
      "loss": 0.1563,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7021253108978271,
      "rewards/margins": 4.240569591522217,
      "rewards/rejected": -5.942694664001465,
      "step": 1131
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.7289919115997374e-05,
      "logits/chosen": -1.9014203548431396,
      "logits/rejected": -1.9022771120071411,
      "logps/chosen": -150.83734130859375,
      "logps/rejected": -227.0953369140625,
      "loss": 0.1021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6051936149597168,
      "rewards/margins": 7.756526947021484,
      "rewards/rejected": -9.36172103881836,
      "step": 1132
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.7254237191917776e-05,
      "logits/chosen": -1.4715416431427002,
      "logits/rejected": -1.6267719268798828,
      "logps/chosen": -189.58242797851562,
      "logps/rejected": -286.1520690917969,
      "loss": 0.0079,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41883718967437744,
      "rewards/margins": 9.405654907226562,
      "rewards/rejected": -9.824492454528809,
      "step": 1133
    },
    {
      "epoch": 1.48,
      "learning_rate": 2.721855063744165e-05,
      "logits/chosen": -1.6812471151351929,
      "logits/rejected": -1.6767619848251343,
      "logps/chosen": -150.61895751953125,
      "logps/rejected": -238.74180603027344,
      "loss": 0.0202,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.18735408782959,
      "rewards/margins": 7.309322357177734,
      "rewards/rejected": -9.496676445007324,
      "step": 1134
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.718285952587228e-05,
      "logits/chosen": -1.9848941564559937,
      "logits/rejected": -2.020820140838623,
      "logps/chosen": -218.78622436523438,
      "logps/rejected": -327.712890625,
      "loss": 0.0494,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3604309558868408,
      "rewards/margins": 9.434850692749023,
      "rewards/rejected": -10.795280456542969,
      "step": 1135
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.714716393052223e-05,
      "logits/chosen": -1.6857093572616577,
      "logits/rejected": -1.582934856414795,
      "logps/chosen": -180.24562072753906,
      "logps/rejected": -235.8282928466797,
      "loss": 0.1073,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0078940391540527,
      "rewards/margins": 6.802659034729004,
      "rewards/rejected": -8.810553550720215,
      "step": 1136
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.711146392471333e-05,
      "logits/chosen": -1.8224282264709473,
      "logits/rejected": -1.7951526641845703,
      "logps/chosen": -174.60769653320312,
      "logps/rejected": -249.1178741455078,
      "loss": 0.0792,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0923131704330444,
      "rewards/margins": 8.285701751708984,
      "rewards/rejected": -9.378015518188477,
      "step": 1137
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.7075759581776462e-05,
      "logits/chosen": -1.832988977432251,
      "logits/rejected": -1.828893780708313,
      "logps/chosen": -182.28993225097656,
      "logps/rejected": -231.23463439941406,
      "loss": 0.0644,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5688951015472412,
      "rewards/margins": 6.473936080932617,
      "rewards/rejected": -8.042831420898438,
      "step": 1138
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.704005097505139e-05,
      "logits/chosen": -1.8686728477478027,
      "logits/rejected": -1.8990896940231323,
      "logps/chosen": -181.93399047851562,
      "logps/rejected": -259.093505859375,
      "loss": 0.0482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7485432624816895,
      "rewards/margins": 7.390007972717285,
      "rewards/rejected": -9.138550758361816,
      "step": 1139
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.7004338177886672e-05,
      "logits/chosen": -1.6753220558166504,
      "logits/rejected": -1.7323517799377441,
      "logps/chosen": -195.35708618164062,
      "logps/rejected": -269.27276611328125,
      "loss": 0.0256,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2014330625534058,
      "rewards/margins": 8.885977745056152,
      "rewards/rejected": -10.087409973144531,
      "step": 1140
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.6968621263639444e-05,
      "logits/chosen": -1.5731022357940674,
      "logits/rejected": -1.6291849613189697,
      "logps/chosen": -155.51751708984375,
      "logps/rejected": -248.51417541503906,
      "loss": 0.0494,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9757877588272095,
      "rewards/margins": 7.520580768585205,
      "rewards/rejected": -8.496369361877441,
      "step": 1141
    },
    {
      "epoch": 1.49,
      "learning_rate": 2.693290030567532e-05,
      "logits/chosen": -1.7571489810943604,
      "logits/rejected": -1.843751311302185,
      "logps/chosen": -213.80398559570312,
      "logps/rejected": -300.8934326171875,
      "loss": 0.097,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2863292694091797,
      "rewards/margins": 7.1869282722473145,
      "rewards/rejected": -9.473258018493652,
      "step": 1142
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.6897175377368207e-05,
      "logits/chosen": -1.50496244430542,
      "logits/rejected": -1.5212422609329224,
      "logps/chosen": -157.7215118408203,
      "logps/rejected": -193.2350311279297,
      "loss": 0.1837,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.29038143157959,
      "rewards/margins": 4.409356117248535,
      "rewards/rejected": -6.699736595153809,
      "step": 1143
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.686144655210016e-05,
      "logits/chosen": -1.8769636154174805,
      "logits/rejected": -1.9429951906204224,
      "logps/chosen": -166.583251953125,
      "logps/rejected": -249.98330688476562,
      "loss": 0.0472,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.788864254951477,
      "rewards/margins": 7.811670303344727,
      "rewards/rejected": -9.600534439086914,
      "step": 1144
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.6825713903261273e-05,
      "logits/chosen": -1.897660732269287,
      "logits/rejected": -1.8566992282867432,
      "logps/chosen": -188.60549926757812,
      "logps/rejected": -253.7261962890625,
      "loss": 0.1837,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9235419034957886,
      "rewards/margins": 5.652706623077393,
      "rewards/rejected": -7.5762481689453125,
      "step": 1145
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.6789977504249454e-05,
      "logits/chosen": -1.94535493850708,
      "logits/rejected": -1.9822051525115967,
      "logps/chosen": -149.48272705078125,
      "logps/rejected": -225.78135681152344,
      "loss": 0.148,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.4285035133361816,
      "rewards/margins": 5.41309928894043,
      "rewards/rejected": -7.841601848602295,
      "step": 1146
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.6754237428470336e-05,
      "logits/chosen": -1.6976293325424194,
      "logits/rejected": -1.586763858795166,
      "logps/chosen": -167.99774169921875,
      "logps/rejected": -268.14984130859375,
      "loss": 0.0275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1044127941131592,
      "rewards/margins": 8.773801803588867,
      "rewards/rejected": -9.878213882446289,
      "step": 1147
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.6718493749337105e-05,
      "logits/chosen": -1.8314532041549683,
      "logits/rejected": -1.8665634393692017,
      "logps/chosen": -164.98809814453125,
      "logps/rejected": -219.7435760498047,
      "loss": 0.0552,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4525818824768066,
      "rewards/margins": 5.477141380310059,
      "rewards/rejected": -7.929723739624023,
      "step": 1148
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.668274654027033e-05,
      "logits/chosen": -1.9173221588134766,
      "logits/rejected": -1.9428825378417969,
      "logps/chosen": -168.40011596679688,
      "logps/rejected": -269.0062255859375,
      "loss": 0.0355,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6917105317115784,
      "rewards/margins": 9.441150665283203,
      "rewards/rejected": -10.13286018371582,
      "step": 1149
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.664699587469786e-05,
      "logits/chosen": -1.6640487909317017,
      "logits/rejected": -1.6672810316085815,
      "logps/chosen": -178.23056030273438,
      "logps/rejected": -235.25369262695312,
      "loss": 0.0919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9629117846488953,
      "rewards/margins": 6.23097562789917,
      "rewards/rejected": -7.193887710571289,
      "step": 1150
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.6611241826054617e-05,
      "logits/chosen": -1.6758863925933838,
      "logits/rejected": -1.6919294595718384,
      "logps/chosen": -141.7913818359375,
      "logps/rejected": -214.95664978027344,
      "loss": 0.1762,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.7705963850021362,
      "rewards/margins": 7.193760871887207,
      "rewards/rejected": -7.964357376098633,
      "step": 1151
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.6575484467782486e-05,
      "logits/chosen": -1.7097420692443848,
      "logits/rejected": -1.7328468561172485,
      "logps/chosen": -165.11996459960938,
      "logps/rejected": -227.81808471679688,
      "loss": 0.0851,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.827000379562378,
      "rewards/margins": 6.5736002922058105,
      "rewards/rejected": -8.40060043334961,
      "step": 1152
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.6539723873330148e-05,
      "logits/chosen": -1.8107922077178955,
      "logits/rejected": -1.816512942314148,
      "logps/chosen": -146.89158630371094,
      "logps/rejected": -241.9942169189453,
      "loss": 0.0567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.013214349746704,
      "rewards/margins": 7.304594039916992,
      "rewards/rejected": -9.317808151245117,
      "step": 1153
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.6503960116152933e-05,
      "logits/chosen": -1.8430949449539185,
      "logits/rejected": -1.9367519617080688,
      "logps/chosen": -178.0413818359375,
      "logps/rejected": -256.8725280761719,
      "loss": 0.151,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7391347289085388,
      "rewards/margins": 7.840487480163574,
      "rewards/rejected": -8.579622268676758,
      "step": 1154
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.646819326971266e-05,
      "logits/chosen": -1.7391657829284668,
      "logits/rejected": -1.774857997894287,
      "logps/chosen": -156.6804962158203,
      "logps/rejected": -230.7911834716797,
      "loss": 0.1137,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3989520072937012,
      "rewards/margins": 6.06267786026001,
      "rewards/rejected": -7.461629867553711,
      "step": 1155
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.6432423407477496e-05,
      "logits/chosen": -1.720785140991211,
      "logits/rejected": -1.9792776107788086,
      "logps/chosen": -136.76864624023438,
      "logps/rejected": -231.67526245117188,
      "loss": 0.0303,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8357769250869751,
      "rewards/margins": 7.876999855041504,
      "rewards/rejected": -8.712776184082031,
      "step": 1156
    },
    {
      "epoch": 1.51,
      "learning_rate": 2.6396650602921824e-05,
      "logits/chosen": -1.6088004112243652,
      "logits/rejected": -1.517633318901062,
      "logps/chosen": -195.29269409179688,
      "logps/rejected": -293.4590148925781,
      "loss": 0.0646,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1006734371185303,
      "rewards/margins": 7.843051433563232,
      "rewards/rejected": -9.943723678588867,
      "step": 1157
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.636087492952603e-05,
      "logits/chosen": -1.6810226440429688,
      "logits/rejected": -1.577724814414978,
      "logps/chosen": -187.27175903320312,
      "logps/rejected": -230.37969970703125,
      "loss": 0.0643,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8785756826400757,
      "rewards/margins": 5.93630838394165,
      "rewards/rejected": -7.814884185791016,
      "step": 1158
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.6325096460776422e-05,
      "logits/chosen": -1.8230814933776855,
      "logits/rejected": -1.7935919761657715,
      "logps/chosen": -192.18484497070312,
      "logps/rejected": -271.8443603515625,
      "loss": 0.0555,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0221341848373413,
      "rewards/margins": 7.83449125289917,
      "rewards/rejected": -8.8566255569458,
      "step": 1159
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.6289315270165062e-05,
      "logits/chosen": -1.7090122699737549,
      "logits/rejected": -1.6645526885986328,
      "logps/chosen": -180.00071716308594,
      "logps/rejected": -277.3699035644531,
      "loss": 0.0611,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2371472269296646,
      "rewards/margins": 8.0440092086792,
      "rewards/rejected": -8.281156539916992,
      "step": 1160
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.625353143118955e-05,
      "logits/chosen": -1.8427613973617554,
      "logits/rejected": -1.8507212400436401,
      "logps/chosen": -164.959228515625,
      "logps/rejected": -225.01565551757812,
      "loss": 0.113,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6578110456466675,
      "rewards/margins": 5.629616737365723,
      "rewards/rejected": -7.28742790222168,
      "step": 1161
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.621774501735299e-05,
      "logits/chosen": -2.031158208847046,
      "logits/rejected": -2.012693405151367,
      "logps/chosen": -160.92926025390625,
      "logps/rejected": -227.06219482421875,
      "loss": 0.075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1299777030944824,
      "rewards/margins": 5.8114423751831055,
      "rewards/rejected": -7.941420555114746,
      "step": 1162
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.6181956102163724e-05,
      "logits/chosen": -1.9714467525482178,
      "logits/rejected": -1.9419063329696655,
      "logps/chosen": -166.3640899658203,
      "logps/rejected": -224.0509796142578,
      "loss": 0.0501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3773655891418457,
      "rewards/margins": 6.627005100250244,
      "rewards/rejected": -8.00437068939209,
      "step": 1163
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.6146164759135266e-05,
      "logits/chosen": -1.7344276905059814,
      "logits/rejected": -1.7813503742218018,
      "logps/chosen": -192.6088104248047,
      "logps/rejected": -254.84864807128906,
      "loss": 0.0611,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.679912567138672,
      "rewards/margins": 6.474228382110596,
      "rewards/rejected": -9.15414047241211,
      "step": 1164
    },
    {
      "epoch": 1.52,
      "learning_rate": 2.6110371061786104e-05,
      "logits/chosen": -1.5994207859039307,
      "logits/rejected": -1.6428487300872803,
      "logps/chosen": -151.71339416503906,
      "logps/rejected": -250.27981567382812,
      "loss": 0.0712,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.336500883102417,
      "rewards/margins": 8.175589561462402,
      "rewards/rejected": -9.512090682983398,
      "step": 1165
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.607457508363955e-05,
      "logits/chosen": -1.8259451389312744,
      "logits/rejected": -1.8387179374694824,
      "logps/chosen": -187.05023193359375,
      "logps/rejected": -240.42816162109375,
      "loss": 0.1209,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.471937656402588,
      "rewards/margins": 5.849018573760986,
      "rewards/rejected": -8.32095718383789,
      "step": 1166
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.6038776898223627e-05,
      "logits/chosen": -1.6083053350448608,
      "logits/rejected": -1.647005558013916,
      "logps/chosen": -172.984619140625,
      "logps/rejected": -291.38287353515625,
      "loss": 0.0532,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1960625648498535,
      "rewards/margins": 7.705469131469727,
      "rewards/rejected": -9.901531219482422,
      "step": 1167
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.6002976579070872e-05,
      "logits/chosen": -1.9568921327590942,
      "logits/rejected": -1.9520502090454102,
      "logps/chosen": -157.1878204345703,
      "logps/rejected": -229.75067138671875,
      "loss": 0.0622,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4645514488220215,
      "rewards/margins": 7.039254188537598,
      "rewards/rejected": -8.503806114196777,
      "step": 1168
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.5967174199718202e-05,
      "logits/chosen": -1.7094905376434326,
      "logits/rejected": -1.764892578125,
      "logps/chosen": -138.9808349609375,
      "logps/rejected": -232.25180053710938,
      "loss": 0.0356,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.58858060836792,
      "rewards/margins": 7.593311309814453,
      "rewards/rejected": -9.181891441345215,
      "step": 1169
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.5931369833706797e-05,
      "logits/chosen": -1.890617847442627,
      "logits/rejected": -1.8630344867706299,
      "logps/chosen": -266.2156066894531,
      "logps/rejected": -310.5943603515625,
      "loss": 0.1574,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5044243335723877,
      "rewards/margins": 6.97006368637085,
      "rewards/rejected": -8.474488258361816,
      "step": 1170
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.5895563554581865e-05,
      "logits/chosen": -1.764174222946167,
      "logits/rejected": -1.7898168563842773,
      "logps/chosen": -195.39540100097656,
      "logps/rejected": -264.393798828125,
      "loss": 0.0825,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9337760210037231,
      "rewards/margins": 6.867251396179199,
      "rewards/rejected": -8.801027297973633,
      "step": 1171
    },
    {
      "epoch": 1.53,
      "learning_rate": 2.5859755435892597e-05,
      "logits/chosen": -1.9995322227478027,
      "logits/rejected": -1.8863884210586548,
      "logps/chosen": -172.40310668945312,
      "logps/rejected": -239.08843994140625,
      "loss": 0.0548,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.442542314529419,
      "rewards/margins": 5.999819278717041,
      "rewards/rejected": -7.442361831665039,
      "step": 1172
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.5823945551191937e-05,
      "logits/chosen": -1.8315728902816772,
      "logits/rejected": -1.871559977531433,
      "logps/chosen": -193.8673553466797,
      "logps/rejected": -228.833251953125,
      "loss": 0.1078,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.922398030757904,
      "rewards/margins": 5.9219970703125,
      "rewards/rejected": -6.844394683837891,
      "step": 1173
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.578813397403645e-05,
      "logits/chosen": -1.665716528892517,
      "logits/rejected": -1.7428711652755737,
      "logps/chosen": -173.10537719726562,
      "logps/rejected": -302.48907470703125,
      "loss": 0.035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5985413789749146,
      "rewards/margins": 8.09644603729248,
      "rewards/rejected": -9.694986343383789,
      "step": 1174
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.5752320777986195e-05,
      "logits/chosen": -1.9972821474075317,
      "logits/rejected": -1.9924802780151367,
      "logps/chosen": -169.9102325439453,
      "logps/rejected": -241.30776977539062,
      "loss": 0.0799,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.378887414932251,
      "rewards/margins": 7.668159484863281,
      "rewards/rejected": -9.04704761505127,
      "step": 1175
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.5716506036604542e-05,
      "logits/chosen": -1.8974864482879639,
      "logits/rejected": -1.873181939125061,
      "logps/chosen": -153.9471435546875,
      "logps/rejected": -224.0230712890625,
      "loss": 0.0525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2690376043319702,
      "rewards/margins": 6.553219318389893,
      "rewards/rejected": -7.8222575187683105,
      "step": 1176
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.568068982345804e-05,
      "logits/chosen": -1.708647608757019,
      "logits/rejected": -1.8205957412719727,
      "logps/chosen": -242.8329620361328,
      "logps/rejected": -301.3001403808594,
      "loss": 0.0542,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.41377592086792,
      "rewards/margins": 6.60507869720459,
      "rewards/rejected": -8.018854141235352,
      "step": 1177
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.5644872212116267e-05,
      "logits/chosen": -1.8445649147033691,
      "logits/rejected": -1.803357720375061,
      "logps/chosen": -183.392578125,
      "logps/rejected": -240.37001037597656,
      "loss": 0.0146,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3879624605178833,
      "rewards/margins": 6.334040641784668,
      "rewards/rejected": -7.722002983093262,
      "step": 1178
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.560905327615168e-05,
      "logits/chosen": -1.7166202068328857,
      "logits/rejected": -1.6495661735534668,
      "logps/chosen": -158.63047790527344,
      "logps/rejected": -211.68258666992188,
      "loss": 0.0295,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6046478748321533,
      "rewards/margins": 6.967181205749512,
      "rewards/rejected": -8.571828842163086,
      "step": 1179
    },
    {
      "epoch": 1.54,
      "learning_rate": 2.557323308913942e-05,
      "logits/chosen": -1.8683463335037231,
      "logits/rejected": -1.8754944801330566,
      "logps/chosen": -167.97018432617188,
      "logps/rejected": -257.2234191894531,
      "loss": 0.036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6898072361946106,
      "rewards/margins": 7.153036117553711,
      "rewards/rejected": -7.842843532562256,
      "step": 1180
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.553741172465724e-05,
      "logits/chosen": -2.0570003986358643,
      "logits/rejected": -2.0702872276306152,
      "logps/chosen": -153.0985107421875,
      "logps/rejected": -233.74180603027344,
      "loss": 0.1304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2187212705612183,
      "rewards/margins": 7.32631778717041,
      "rewards/rejected": -8.545040130615234,
      "step": 1181
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.5501589256285285e-05,
      "logits/chosen": -1.5223973989486694,
      "logits/rejected": -1.5868252515792847,
      "logps/chosen": -156.3800811767578,
      "logps/rejected": -242.82012939453125,
      "loss": 0.12,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6241587400436401,
      "rewards/margins": 5.533423900604248,
      "rewards/rejected": -7.157582759857178,
      "step": 1182
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.546576575760598e-05,
      "logits/chosen": -1.9983586072921753,
      "logits/rejected": -2.0023610591888428,
      "logps/chosen": -170.19078063964844,
      "logps/rejected": -275.523193359375,
      "loss": 0.0062,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4514535665512085,
      "rewards/margins": 10.039039611816406,
      "rewards/rejected": -10.490493774414062,
      "step": 1183
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.542994130220388e-05,
      "logits/chosen": -1.6396501064300537,
      "logits/rejected": -1.7652050256729126,
      "logps/chosen": -160.75619506835938,
      "logps/rejected": -265.81756591796875,
      "loss": 0.0413,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0278128385543823,
      "rewards/margins": 8.429641723632812,
      "rewards/rejected": -9.457454681396484,
      "step": 1184
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.539411596366546e-05,
      "logits/chosen": -1.8460824489593506,
      "logits/rejected": -1.9252185821533203,
      "logps/chosen": -166.89138793945312,
      "logps/rejected": -248.66616821289062,
      "loss": 0.0997,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6275098323822021,
      "rewards/margins": 6.504864692687988,
      "rewards/rejected": -8.132373809814453,
      "step": 1185
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.535828981557906e-05,
      "logits/chosen": -2.0881285667419434,
      "logits/rejected": -2.042153835296631,
      "logps/chosen": -177.7544403076172,
      "logps/rejected": -232.4076385498047,
      "loss": 0.1317,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8214682340621948,
      "rewards/margins": 6.011728286743164,
      "rewards/rejected": -7.833196640014648,
      "step": 1186
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.5322462931534658e-05,
      "logits/chosen": -1.7492239475250244,
      "logits/rejected": -1.8416800498962402,
      "logps/chosen": -136.50247192382812,
      "logps/rejected": -208.47909545898438,
      "loss": 0.1289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9211529493331909,
      "rewards/margins": 6.29665470123291,
      "rewards/rejected": -7.217808723449707,
      "step": 1187
    },
    {
      "epoch": 1.55,
      "learning_rate": 2.5286635385123725e-05,
      "logits/chosen": -1.749982476234436,
      "logits/rejected": -1.8284657001495361,
      "logps/chosen": -155.00045776367188,
      "logps/rejected": -238.42721557617188,
      "loss": 0.1181,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8909995555877686,
      "rewards/margins": 6.114850044250488,
      "rewards/rejected": -8.005849838256836,
      "step": 1188
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.525080724993914e-05,
      "logits/chosen": -1.6124377250671387,
      "logits/rejected": -1.6236196756362915,
      "logps/chosen": -167.70858764648438,
      "logps/rejected": -225.40110778808594,
      "loss": 0.0977,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8953793048858643,
      "rewards/margins": 6.390057563781738,
      "rewards/rejected": -8.28543758392334,
      "step": 1189
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.521497859957495e-05,
      "logits/chosen": -1.887227177619934,
      "logits/rejected": -1.8924245834350586,
      "logps/chosen": -169.80819702148438,
      "logps/rejected": -230.32925415039062,
      "loss": 0.1834,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9796323776245117,
      "rewards/margins": 5.563421249389648,
      "rewards/rejected": -7.543054103851318,
      "step": 1190
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.5179149507626288e-05,
      "logits/chosen": -1.941737174987793,
      "logits/rejected": -1.9230965375900269,
      "logps/chosen": -168.2054443359375,
      "logps/rejected": -275.2392272949219,
      "loss": 0.0121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8135576248168945,
      "rewards/margins": 9.723443984985352,
      "rewards/rejected": -10.537002563476562,
      "step": 1191
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.5143320047689173e-05,
      "logits/chosen": -2.0912604331970215,
      "logits/rejected": -2.077686071395874,
      "logps/chosen": -190.74667358398438,
      "logps/rejected": -244.5761260986328,
      "loss": 0.028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6529235243797302,
      "rewards/margins": 5.860649585723877,
      "rewards/rejected": -6.513573169708252,
      "step": 1192
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.510749029336038e-05,
      "logits/chosen": -1.7814724445343018,
      "logits/rejected": -1.715226173400879,
      "logps/chosen": -152.57223510742188,
      "logps/rejected": -188.431640625,
      "loss": 0.0759,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4422708749771118,
      "rewards/margins": 4.994534969329834,
      "rewards/rejected": -6.4368062019348145,
      "step": 1193
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.5071660318237312e-05,
      "logits/chosen": -1.592630386352539,
      "logits/rejected": -1.6790491342544556,
      "logps/chosen": -175.57151794433594,
      "logps/rejected": -242.02462768554688,
      "loss": 0.0436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3934221267700195,
      "rewards/margins": 5.979648590087891,
      "rewards/rejected": -8.37307071685791,
      "step": 1194
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.5035830195917803e-05,
      "logits/chosen": -1.843570351600647,
      "logits/rejected": -1.8633430004119873,
      "logps/chosen": -163.16200256347656,
      "logps/rejected": -265.57330322265625,
      "loss": 0.0469,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9609366655349731,
      "rewards/margins": 8.519161224365234,
      "rewards/rejected": -9.480098724365234,
      "step": 1195
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.5e-05,
      "logits/chosen": -1.864723563194275,
      "logits/rejected": -1.8295310735702515,
      "logps/chosen": -170.7367706298828,
      "logps/rejected": -288.0226745605469,
      "loss": 0.0328,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7146817445755005,
      "rewards/margins": 9.274417877197266,
      "rewards/rejected": -9.989100456237793,
      "step": 1196
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.49641698040822e-05,
      "logits/chosen": -1.8404507637023926,
      "logits/rejected": -1.7767809629440308,
      "logps/chosen": -197.76651000976562,
      "logps/rejected": -232.49969482421875,
      "loss": 0.1316,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7037895917892456,
      "rewards/margins": 4.999301910400391,
      "rewards/rejected": -6.703091621398926,
      "step": 1197
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.4928339681762687e-05,
      "logits/chosen": -2.103551149368286,
      "logits/rejected": -2.0053913593292236,
      "logps/chosen": -242.35336303710938,
      "logps/rejected": -260.0535888671875,
      "loss": 0.2947,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.956660747528076,
      "rewards/margins": 3.049253225326538,
      "rewards/rejected": -6.005914211273193,
      "step": 1198
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.489250970663963e-05,
      "logits/chosen": -1.9753344058990479,
      "logits/rejected": -1.9782427549362183,
      "logps/chosen": -189.11727905273438,
      "logps/rejected": -252.70472717285156,
      "loss": 0.0418,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2426655292510986,
      "rewards/margins": 7.71958065032959,
      "rewards/rejected": -8.962246894836426,
      "step": 1199
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.485667995231084e-05,
      "logits/chosen": -2.1043999195098877,
      "logits/rejected": -2.071964740753174,
      "logps/chosen": -161.5220947265625,
      "logps/rejected": -216.64004516601562,
      "loss": 0.2,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.743095338344574,
      "rewards/margins": 5.800675392150879,
      "rewards/rejected": -6.5437703132629395,
      "step": 1200
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.4820850492373718e-05,
      "logits/chosen": -1.9000133275985718,
      "logits/rejected": -1.893449306488037,
      "logps/chosen": -146.74574279785156,
      "logps/rejected": -199.89437866210938,
      "loss": 0.155,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1161205768585205,
      "rewards/margins": 5.4206414222717285,
      "rewards/rejected": -6.536762714385986,
      "step": 1201
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.4785021400425053e-05,
      "logits/chosen": -1.9871101379394531,
      "logits/rejected": -2.053927183151245,
      "logps/chosen": -160.19610595703125,
      "logps/rejected": -228.89332580566406,
      "loss": 0.075,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5364221334457397,
      "rewards/margins": 5.996157646179199,
      "rewards/rejected": -7.53257942199707,
      "step": 1202
    },
    {
      "epoch": 1.57,
      "learning_rate": 2.474919275006086e-05,
      "logits/chosen": -1.9598592519760132,
      "logits/rejected": -1.992692232131958,
      "logps/chosen": -161.35491943359375,
      "logps/rejected": -249.83917236328125,
      "loss": 0.2099,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.279801845550537,
      "rewards/margins": 6.798189163208008,
      "rewards/rejected": -8.077991485595703,
      "step": 1203
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.4713364614876274e-05,
      "logits/chosen": -2.004483699798584,
      "logits/rejected": -1.9668254852294922,
      "logps/chosen": -205.65469360351562,
      "logps/rejected": -265.10015869140625,
      "loss": 0.0546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9826128482818604,
      "rewards/margins": 6.3379225730896,
      "rewards/rejected": -7.320534706115723,
      "step": 1204
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.4677537068465355e-05,
      "logits/chosen": -1.661895751953125,
      "logits/rejected": -1.785185694694519,
      "logps/chosen": -157.33680725097656,
      "logps/rejected": -238.50738525390625,
      "loss": 0.0414,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2065367698669434,
      "rewards/margins": 7.05164098739624,
      "rewards/rejected": -8.258177757263184,
      "step": 1205
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.4641710184420945e-05,
      "logits/chosen": -1.757274866104126,
      "logits/rejected": -1.7770092487335205,
      "logps/chosen": -142.01470947265625,
      "logps/rejected": -202.0991973876953,
      "loss": 0.0501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6882623434066772,
      "rewards/margins": 6.090262413024902,
      "rewards/rejected": -6.778524398803711,
      "step": 1206
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.4605884036334546e-05,
      "logits/chosen": -1.967075228691101,
      "logits/rejected": -1.9980759620666504,
      "logps/chosen": -149.2307891845703,
      "logps/rejected": -235.4240264892578,
      "loss": 0.223,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8612712621688843,
      "rewards/margins": 6.660323619842529,
      "rewards/rejected": -7.5215959548950195,
      "step": 1207
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.4570058697796125e-05,
      "logits/chosen": -1.6633689403533936,
      "logits/rejected": -1.703428030014038,
      "logps/chosen": -159.02630615234375,
      "logps/rejected": -274.7862243652344,
      "loss": 0.0088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8695993423461914,
      "rewards/margins": 9.023956298828125,
      "rewards/rejected": -9.893556594848633,
      "step": 1208
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.4534234242394015e-05,
      "logits/chosen": -1.918900966644287,
      "logits/rejected": -2.014270544052124,
      "logps/chosen": -178.01400756835938,
      "logps/rejected": -244.51718139648438,
      "loss": 0.0942,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3974378108978271,
      "rewards/margins": 6.708617210388184,
      "rewards/rejected": -8.106056213378906,
      "step": 1209
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.449841074371472e-05,
      "logits/chosen": -1.905793309211731,
      "logits/rejected": -1.894994854927063,
      "logps/chosen": -165.8264617919922,
      "logps/rejected": -229.46051025390625,
      "loss": 0.1022,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4672647714614868,
      "rewards/margins": 6.540958404541016,
      "rewards/rejected": -8.008223533630371,
      "step": 1210
    },
    {
      "epoch": 1.58,
      "learning_rate": 2.4462588275342773e-05,
      "logits/chosen": -2.053560495376587,
      "logits/rejected": -2.086168050765991,
      "logps/chosen": -198.8894805908203,
      "logps/rejected": -248.72079467773438,
      "loss": 0.2289,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2635715007781982,
      "rewards/margins": 4.821905136108398,
      "rewards/rejected": -6.085475921630859,
      "step": 1211
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.4426766910860585e-05,
      "logits/chosen": -1.975711703300476,
      "logits/rejected": -1.9563323259353638,
      "logps/chosen": -172.92947387695312,
      "logps/rejected": -229.34246826171875,
      "loss": 0.0405,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1585272550582886,
      "rewards/margins": 6.81801176071167,
      "rewards/rejected": -7.976539611816406,
      "step": 1212
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.439094672384833e-05,
      "logits/chosen": -1.701197624206543,
      "logits/rejected": -1.6263911724090576,
      "logps/chosen": -176.81777954101562,
      "logps/rejected": -224.1560821533203,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.018692627549171448,
      "rewards/margins": 7.292651653289795,
      "rewards/rejected": -7.273959159851074,
      "step": 1213
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.4355127787883732e-05,
      "logits/chosen": -1.7987830638885498,
      "logits/rejected": -1.8361139297485352,
      "logps/chosen": -162.73475646972656,
      "logps/rejected": -214.76763916015625,
      "loss": 0.11,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9397517442703247,
      "rewards/margins": 6.087156295776367,
      "rewards/rejected": -7.0269083976745605,
      "step": 1214
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.4319310176541958e-05,
      "logits/chosen": -2.0998494625091553,
      "logits/rejected": -2.114942789077759,
      "logps/chosen": -140.2357635498047,
      "logps/rejected": -229.4926300048828,
      "loss": 0.1446,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8301968574523926,
      "rewards/margins": 7.534490585327148,
      "rewards/rejected": -8.364686965942383,
      "step": 1215
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.428349396339547e-05,
      "logits/chosen": -2.0778238773345947,
      "logits/rejected": -2.09785532951355,
      "logps/chosen": -149.57940673828125,
      "logps/rejected": -208.46572875976562,
      "loss": 0.1249,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4522473216056824,
      "rewards/margins": 6.362789630889893,
      "rewards/rejected": -6.815036773681641,
      "step": 1216
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.424767922201381e-05,
      "logits/chosen": -2.0641913414001465,
      "logits/rejected": -2.0436298847198486,
      "logps/chosen": -191.5172882080078,
      "logps/rejected": -258.9608459472656,
      "loss": 0.0446,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4021503925323486,
      "rewards/margins": 6.717792510986328,
      "rewards/rejected": -8.119942665100098,
      "step": 1217
    },
    {
      "epoch": 1.59,
      "learning_rate": 2.4211866025963557e-05,
      "logits/chosen": -1.7434533834457397,
      "logits/rejected": -1.7594411373138428,
      "logps/chosen": -176.60330200195312,
      "logps/rejected": -262.79339599609375,
      "loss": 0.0598,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2568025588989258,
      "rewards/margins": 7.517066955566406,
      "rewards/rejected": -8.773869514465332,
      "step": 1218
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.417605444880807e-05,
      "logits/chosen": -1.753713607788086,
      "logits/rejected": -1.7919538021087646,
      "logps/chosen": -162.9635772705078,
      "logps/rejected": -218.07363891601562,
      "loss": 0.1865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.9248265027999878,
      "rewards/margins": 5.4567036628723145,
      "rewards/rejected": -7.381529808044434,
      "step": 1219
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.4140244564107402e-05,
      "logits/chosen": -1.7603093385696411,
      "logits/rejected": -1.8463753461837769,
      "logps/chosen": -185.3073272705078,
      "logps/rejected": -257.0505676269531,
      "loss": 0.0525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4835914671421051,
      "rewards/margins": 8.16936206817627,
      "rewards/rejected": -8.6529541015625,
      "step": 1220
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.4104436445418145e-05,
      "logits/chosen": -1.8057531118392944,
      "logits/rejected": -1.7772032022476196,
      "logps/chosen": -171.76866149902344,
      "logps/rejected": -233.33737182617188,
      "loss": 0.0883,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3442683219909668,
      "rewards/margins": 6.569925785064697,
      "rewards/rejected": -7.914194107055664,
      "step": 1221
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.4068630166293215e-05,
      "logits/chosen": -1.6516447067260742,
      "logits/rejected": -1.622062087059021,
      "logps/chosen": -152.10487365722656,
      "logps/rejected": -215.98085021972656,
      "loss": 0.1096,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.47995948791503906,
      "rewards/margins": 6.548527717590332,
      "rewards/rejected": -7.028487205505371,
      "step": 1222
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.4032825800281804e-05,
      "logits/chosen": -1.9345694780349731,
      "logits/rejected": -1.9134182929992676,
      "logps/chosen": -141.57325744628906,
      "logps/rejected": -211.09896850585938,
      "loss": 0.0713,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2298121452331543,
      "rewards/margins": 6.853342533111572,
      "rewards/rejected": -8.083155632019043,
      "step": 1223
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.3997023420929137e-05,
      "logits/chosen": -1.8254691362380981,
      "logits/rejected": -1.8211302757263184,
      "logps/chosen": -163.06524658203125,
      "logps/rejected": -225.7220001220703,
      "loss": 0.0264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6015232801437378,
      "rewards/margins": 6.818352222442627,
      "rewards/rejected": -8.419876098632812,
      "step": 1224
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.3961223101776375e-05,
      "logits/chosen": -1.9663257598876953,
      "logits/rejected": -1.9120805263519287,
      "logps/chosen": -161.02044677734375,
      "logps/rejected": -230.92030334472656,
      "loss": 0.0938,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4916810095310211,
      "rewards/margins": 7.360218524932861,
      "rewards/rejected": -7.851898670196533,
      "step": 1225
    },
    {
      "epoch": 1.6,
      "learning_rate": 2.392542491636045e-05,
      "logits/chosen": -1.6812583208084106,
      "logits/rejected": -1.798890233039856,
      "logps/chosen": -145.50582885742188,
      "logps/rejected": -233.9939727783203,
      "loss": 0.015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4378544092178345,
      "rewards/margins": 6.873503684997559,
      "rewards/rejected": -8.311358451843262,
      "step": 1226
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.3889628938213905e-05,
      "logits/chosen": -1.9643669128417969,
      "logits/rejected": -1.9415106773376465,
      "logps/chosen": -146.2119598388672,
      "logps/rejected": -196.58169555664062,
      "loss": 0.1504,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6560534238815308,
      "rewards/margins": 5.058835983276367,
      "rewards/rejected": -6.7148895263671875,
      "step": 1227
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.3853835240864743e-05,
      "logits/chosen": -2.009258985519409,
      "logits/rejected": -2.0232186317443848,
      "logps/chosen": -208.3326873779297,
      "logps/rejected": -297.39434814453125,
      "loss": 0.0538,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.742518961429596,
      "rewards/margins": 7.446087837219238,
      "rewards/rejected": -8.188607215881348,
      "step": 1228
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.381804389783628e-05,
      "logits/chosen": -2.0682835578918457,
      "logits/rejected": -2.0701067447662354,
      "logps/chosen": -161.4124755859375,
      "logps/rejected": -230.60389709472656,
      "loss": 0.1517,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0378248691558838,
      "rewards/margins": 6.704135894775391,
      "rewards/rejected": -7.741960048675537,
      "step": 1229
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.3782254982647013e-05,
      "logits/chosen": -1.9518893957138062,
      "logits/rejected": -1.9690879583358765,
      "logps/chosen": -142.19068908691406,
      "logps/rejected": -219.90255737304688,
      "loss": 0.1341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3289569020271301,
      "rewards/margins": 7.000800132751465,
      "rewards/rejected": -7.329756259918213,
      "step": 1230
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.374646856881045e-05,
      "logits/chosen": -1.8535008430480957,
      "logits/rejected": -1.868861198425293,
      "logps/chosen": -179.60769653320312,
      "logps/rejected": -213.74136352539062,
      "loss": 0.091,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1025397777557373,
      "rewards/margins": 5.429540634155273,
      "rewards/rejected": -6.532081127166748,
      "step": 1231
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.3710684729834954e-05,
      "logits/chosen": -1.9428719282150269,
      "logits/rejected": -1.9820836782455444,
      "logps/chosen": -143.43795776367188,
      "logps/rejected": -259.65130615234375,
      "loss": 0.0122,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5686644911766052,
      "rewards/margins": 9.013404846191406,
      "rewards/rejected": -9.582069396972656,
      "step": 1232
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.367490353922358e-05,
      "logits/chosen": -1.912165641784668,
      "logits/rejected": -1.9195115566253662,
      "logps/chosen": -174.22247314453125,
      "logps/rejected": -235.47195434570312,
      "loss": 0.1455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5298904180526733,
      "rewards/margins": 6.001532077789307,
      "rewards/rejected": -6.531422138214111,
      "step": 1233
    },
    {
      "epoch": 1.61,
      "learning_rate": 2.3639125070473975e-05,
      "logits/chosen": -1.8399956226348877,
      "logits/rejected": -1.932726502418518,
      "logps/chosen": -134.49899291992188,
      "logps/rejected": -230.75640869140625,
      "loss": 0.0224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1378332376480103,
      "rewards/margins": 7.347419738769531,
      "rewards/rejected": -8.48525333404541,
      "step": 1234
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.3603349397078182e-05,
      "logits/chosen": -1.8837718963623047,
      "logits/rejected": -1.95962655544281,
      "logps/chosen": -172.75482177734375,
      "logps/rejected": -226.9110107421875,
      "loss": 0.1182,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4856610298156738,
      "rewards/margins": 5.164188385009766,
      "rewards/rejected": -6.6498494148254395,
      "step": 1235
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.3567576592522507e-05,
      "logits/chosen": -1.8667960166931152,
      "logits/rejected": -1.9278860092163086,
      "logps/chosen": -174.6141815185547,
      "logps/rejected": -250.93017578125,
      "loss": 0.1003,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6973843574523926,
      "rewards/margins": 7.526643753051758,
      "rewards/rejected": -8.224028587341309,
      "step": 1236
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.3531806730287342e-05,
      "logits/chosen": -1.4700721502304077,
      "logits/rejected": -1.445304036140442,
      "logps/chosen": -245.1001739501953,
      "logps/rejected": -295.76275634765625,
      "loss": 0.098,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7860945463180542,
      "rewards/margins": 6.493374347686768,
      "rewards/rejected": -7.279469013214111,
      "step": 1237
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.349603988384708e-05,
      "logits/chosen": -1.695169448852539,
      "logits/rejected": -1.688912034034729,
      "logps/chosen": -202.03501892089844,
      "logps/rejected": -302.9530029296875,
      "loss": 0.0962,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2090994119644165,
      "rewards/margins": 6.75393009185791,
      "rewards/rejected": -7.963029861450195,
      "step": 1238
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.3460276126669854e-05,
      "logits/chosen": -1.6799559593200684,
      "logits/rejected": -1.7453773021697998,
      "logps/chosen": -144.6733856201172,
      "logps/rejected": -254.51611328125,
      "loss": 0.0933,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7565066814422607,
      "rewards/margins": 7.9578857421875,
      "rewards/rejected": -8.71439266204834,
      "step": 1239
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.342451553221752e-05,
      "logits/chosen": -1.847500205039978,
      "logits/rejected": -1.8010544776916504,
      "logps/chosen": -175.72085571289062,
      "logps/rejected": -239.20738220214844,
      "loss": 0.0665,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.019901156425476,
      "rewards/margins": 6.560042381286621,
      "rewards/rejected": -7.57994270324707,
      "step": 1240
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.338875817394539e-05,
      "logits/chosen": -1.8564703464508057,
      "logits/rejected": -1.762199878692627,
      "logps/chosen": -154.302978515625,
      "logps/rejected": -209.03366088867188,
      "loss": 0.1626,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8055093288421631,
      "rewards/margins": 5.111129283905029,
      "rewards/rejected": -5.9166388511657715,
      "step": 1241
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.3353004125302142e-05,
      "logits/chosen": -1.7121126651763916,
      "logits/rejected": -1.8166656494140625,
      "logps/chosen": -153.10858154296875,
      "logps/rejected": -218.0070343017578,
      "loss": 0.0689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7316798567771912,
      "rewards/margins": 6.801442623138428,
      "rewards/rejected": -7.5331220626831055,
      "step": 1242
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.331725345972968e-05,
      "logits/chosen": -1.8185333013534546,
      "logits/rejected": -1.811452865600586,
      "logps/chosen": -194.3076934814453,
      "logps/rejected": -260.2732849121094,
      "loss": 0.0514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1511259078979492,
      "rewards/margins": 7.879863739013672,
      "rewards/rejected": -9.030989646911621,
      "step": 1243
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.32815062506629e-05,
      "logits/chosen": -2.0430233478546143,
      "logits/rejected": -2.0219738483428955,
      "logps/chosen": -140.8071746826172,
      "logps/rejected": -196.35304260253906,
      "loss": 0.1513,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5003581643104553,
      "rewards/margins": 6.041768550872803,
      "rewards/rejected": -6.542126655578613,
      "step": 1244
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.3245762571529667e-05,
      "logits/chosen": -1.9518170356750488,
      "logits/rejected": -1.9169788360595703,
      "logps/chosen": -157.82980346679688,
      "logps/rejected": -206.12564086914062,
      "loss": 0.1043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2564760446548462,
      "rewards/margins": 5.682040214538574,
      "rewards/rejected": -6.938515663146973,
      "step": 1245
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.3210022495750552e-05,
      "logits/chosen": -1.737129807472229,
      "logits/rejected": -1.7377147674560547,
      "logps/chosen": -166.2869110107422,
      "logps/rejected": -259.5175476074219,
      "loss": 0.0175,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5067781805992126,
      "rewards/margins": 8.633674621582031,
      "rewards/rejected": -9.14045238494873,
      "step": 1246
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.317428609673873e-05,
      "logits/chosen": -2.185476779937744,
      "logits/rejected": -2.1657662391662598,
      "logps/chosen": -157.8886260986328,
      "logps/rejected": -240.30596923828125,
      "loss": 0.1062,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8809615969657898,
      "rewards/margins": 6.701949119567871,
      "rewards/rejected": -7.582910537719727,
      "step": 1247
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.3138553447899835e-05,
      "logits/chosen": -2.0327000617980957,
      "logits/rejected": -2.034411907196045,
      "logps/chosen": -164.22084045410156,
      "logps/rejected": -241.81996154785156,
      "loss": 0.016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.47778424620628357,
      "rewards/margins": 8.468803405761719,
      "rewards/rejected": -8.946588516235352,
      "step": 1248
    },
    {
      "epoch": 1.63,
      "learning_rate": 2.3102824622631803e-05,
      "logits/chosen": -1.8589757680892944,
      "logits/rejected": -2.03688907623291,
      "logps/chosen": -140.984619140625,
      "logps/rejected": -224.69192504882812,
      "loss": 0.0443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1750152111053467,
      "rewards/margins": 6.264841556549072,
      "rewards/rejected": -7.439857006072998,
      "step": 1249
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.3067099694324686e-05,
      "logits/chosen": -1.815935492515564,
      "logits/rejected": -1.899837851524353,
      "logps/chosen": -151.9332275390625,
      "logps/rejected": -219.5342559814453,
      "loss": 0.0622,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3939430713653564,
      "rewards/margins": 6.310131072998047,
      "rewards/rejected": -7.704073905944824,
      "step": 1250
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.3031378736360562e-05,
      "logits/chosen": -1.7778511047363281,
      "logits/rejected": -1.7813197374343872,
      "logps/chosen": -225.9751739501953,
      "logps/rejected": -278.9739990234375,
      "loss": 0.199,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4317359924316406,
      "rewards/margins": 5.573480606079102,
      "rewards/rejected": -7.0052170753479,
      "step": 1251
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.299566182211333e-05,
      "logits/chosen": -2.0333735942840576,
      "logits/rejected": -1.9940904378890991,
      "logps/chosen": -154.86228942871094,
      "logps/rejected": -192.64404296875,
      "loss": 0.1374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0505996942520142,
      "rewards/margins": 5.135288238525391,
      "rewards/rejected": -6.185888290405273,
      "step": 1252
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.295994902494861e-05,
      "logits/chosen": -2.0112099647521973,
      "logits/rejected": -1.9698991775512695,
      "logps/chosen": -154.24176025390625,
      "logps/rejected": -201.0348358154297,
      "loss": 0.1457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.34839725494384766,
      "rewards/margins": 5.991030216217041,
      "rewards/rejected": -6.339426040649414,
      "step": 1253
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.292424041822355e-05,
      "logits/chosen": -2.122105121612549,
      "logits/rejected": -2.089745283126831,
      "logps/chosen": -158.46597290039062,
      "logps/rejected": -227.99954223632812,
      "loss": 0.0375,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28532734513282776,
      "rewards/margins": 6.927036762237549,
      "rewards/rejected": -7.212364196777344,
      "step": 1254
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.2888536075286675e-05,
      "logits/chosen": -1.9770643711090088,
      "logits/rejected": -2.005664110183716,
      "logps/chosen": -198.68414306640625,
      "logps/rejected": -264.5328674316406,
      "loss": 0.1783,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7044317126274109,
      "rewards/margins": 5.797695159912109,
      "rewards/rejected": -6.502127170562744,
      "step": 1255
    },
    {
      "epoch": 1.64,
      "learning_rate": 2.2852836069477773e-05,
      "logits/chosen": -1.9997928142547607,
      "logits/rejected": -2.0332624912261963,
      "logps/chosen": -167.84512329101562,
      "logps/rejected": -226.86407470703125,
      "loss": 0.1105,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3594387173652649,
      "rewards/margins": 5.6900811195373535,
      "rewards/rejected": -6.049520015716553,
      "step": 1256
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.281714047412773e-05,
      "logits/chosen": -1.701532244682312,
      "logits/rejected": -1.7239773273468018,
      "logps/chosen": -162.87530517578125,
      "logps/rejected": -237.6905517578125,
      "loss": 0.1253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5382217168807983,
      "rewards/margins": 5.034113883972168,
      "rewards/rejected": -6.572335243225098,
      "step": 1257
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.2781449362558347e-05,
      "logits/chosen": -2.044706344604492,
      "logits/rejected": -2.000545024871826,
      "logps/chosen": -165.75294494628906,
      "logps/rejected": -229.44000244140625,
      "loss": 0.0586,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.23104336857795715,
      "rewards/margins": 6.941731929779053,
      "rewards/rejected": -7.172774791717529,
      "step": 1258
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.2745762808082223e-05,
      "logits/chosen": -1.6675219535827637,
      "logits/rejected": -1.652753233909607,
      "logps/chosen": -158.45196533203125,
      "logps/rejected": -229.87872314453125,
      "loss": 0.107,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7921045422554016,
      "rewards/margins": 6.75758171081543,
      "rewards/rejected": -7.549686908721924,
      "step": 1259
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.2710080884002632e-05,
      "logits/chosen": -1.9316892623901367,
      "logits/rejected": -1.9654532670974731,
      "logps/chosen": -121.40196228027344,
      "logps/rejected": -151.48410034179688,
      "loss": 0.2763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.0111302137374878,
      "rewards/margins": 3.2882964611053467,
      "rewards/rejected": -4.299427032470703,
      "step": 1260
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.2674403663613267e-05,
      "logits/chosen": -2.0003440380096436,
      "logits/rejected": -1.9867817163467407,
      "logps/chosen": -153.8340606689453,
      "logps/rejected": -225.6444854736328,
      "loss": 0.0387,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7526621222496033,
      "rewards/margins": 6.773000717163086,
      "rewards/rejected": -7.525662899017334,
      "step": 1261
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.263873122019822e-05,
      "logits/chosen": -1.933740258216858,
      "logits/rejected": -1.9163880348205566,
      "logps/chosen": -179.26239013671875,
      "logps/rejected": -233.81976318359375,
      "loss": 0.1771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.542977511882782,
      "rewards/margins": 5.718654155731201,
      "rewards/rejected": -6.261631011962891,
      "step": 1262
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.2603063627031744e-05,
      "logits/chosen": -2.020321846008301,
      "logits/rejected": -2.0156869888305664,
      "logps/chosen": -156.0040283203125,
      "logps/rejected": -232.4320068359375,
      "loss": 0.0275,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6501021385192871,
      "rewards/margins": 6.472480297088623,
      "rewards/rejected": -7.12258243560791,
      "step": 1263
    },
    {
      "epoch": 1.65,
      "learning_rate": 2.2567400957378132e-05,
      "logits/chosen": -1.62031888961792,
      "logits/rejected": -1.6010981798171997,
      "logps/chosen": -193.51748657226562,
      "logps/rejected": -248.55372619628906,
      "loss": 0.1772,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6310415267944336,
      "rewards/margins": 4.877374649047852,
      "rewards/rejected": -6.508415222167969,
      "step": 1264
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.253174328449158e-05,
      "logits/chosen": -2.1037962436676025,
      "logits/rejected": -2.047747850418091,
      "logps/chosen": -166.24752807617188,
      "logps/rejected": -246.21966552734375,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6143661141395569,
      "rewards/margins": 8.562211990356445,
      "rewards/rejected": -9.176578521728516,
      "step": 1265
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.2496090681615984e-05,
      "logits/chosen": -2.0843794345855713,
      "logits/rejected": -2.0433902740478516,
      "logps/chosen": -156.74758911132812,
      "logps/rejected": -203.4853057861328,
      "loss": 0.0993,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.555661678314209,
      "rewards/margins": 5.7755632400512695,
      "rewards/rejected": -6.331225395202637,
      "step": 1266
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.246044322198486e-05,
      "logits/chosen": -2.011796474456787,
      "logits/rejected": -2.017746686935425,
      "logps/chosen": -168.1431121826172,
      "logps/rejected": -237.1455841064453,
      "loss": 0.1058,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5620946884155273,
      "rewards/margins": 7.451038360595703,
      "rewards/rejected": -8.01313304901123,
      "step": 1267
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.2424800978821146e-05,
      "logits/chosen": -2.09619140625,
      "logits/rejected": -2.096158266067505,
      "logps/chosen": -153.12033081054688,
      "logps/rejected": -208.53268432617188,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.27095508575439453,
      "rewards/margins": 5.667902946472168,
      "rewards/rejected": -5.9388580322265625,
      "step": 1268
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.238916402533706e-05,
      "logits/chosen": -1.7176047563552856,
      "logits/rejected": -1.7082040309906006,
      "logps/chosen": -153.0591583251953,
      "logps/rejected": -225.08627319335938,
      "loss": 0.0749,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.103965401649475,
      "rewards/margins": 7.115612983703613,
      "rewards/rejected": -8.21957778930664,
      "step": 1269
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.235353243473398e-05,
      "logits/chosen": -2.0392508506774902,
      "logits/rejected": -2.0662896633148193,
      "logps/chosen": -152.37026977539062,
      "logps/rejected": -260.37115478515625,
      "loss": 0.0193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26387298107147217,
      "rewards/margins": 8.195239067077637,
      "rewards/rejected": -8.459112167358398,
      "step": 1270
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.231790628020222e-05,
      "logits/chosen": -1.9557806253433228,
      "logits/rejected": -1.8642823696136475,
      "logps/chosen": -156.40708923339844,
      "logps/rejected": -207.2935028076172,
      "loss": 0.0083,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5583537220954895,
      "rewards/margins": 6.730528831481934,
      "rewards/rejected": -7.288883209228516,
      "step": 1271
    },
    {
      "epoch": 1.66,
      "learning_rate": 2.228228563492098e-05,
      "logits/chosen": -1.8270533084869385,
      "logits/rejected": -1.8477699756622314,
      "logps/chosen": -209.7089080810547,
      "logps/rejected": -246.34561157226562,
      "loss": 0.0709,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6850746870040894,
      "rewards/margins": 5.968445301055908,
      "rewards/rejected": -6.653519630432129,
      "step": 1272
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.224667057205811e-05,
      "logits/chosen": -1.8454939126968384,
      "logits/rejected": -1.8808714151382446,
      "logps/chosen": -141.45703125,
      "logps/rejected": -215.69482421875,
      "loss": 0.1916,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1101245880126953,
      "rewards/margins": 5.859930038452148,
      "rewards/rejected": -6.97005558013916,
      "step": 1273
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.2211061164769997e-05,
      "logits/chosen": -1.7888787984848022,
      "logits/rejected": -1.8212321996688843,
      "logps/chosen": -164.64254760742188,
      "logps/rejected": -254.43531799316406,
      "loss": 0.0271,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8234186768531799,
      "rewards/margins": 6.194815635681152,
      "rewards/rejected": -7.0182342529296875,
      "step": 1274
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.2175457486201435e-05,
      "logits/chosen": -2.1381893157958984,
      "logits/rejected": -2.1068222522735596,
      "logps/chosen": -181.55218505859375,
      "logps/rejected": -235.17340087890625,
      "loss": 0.1059,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5667086243629456,
      "rewards/margins": 6.662299633026123,
      "rewards/rejected": -7.22900915145874,
      "step": 1275
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.2139859609485426e-05,
      "logits/chosen": -1.7071415185928345,
      "logits/rejected": -1.7188609838485718,
      "logps/chosen": -169.3489227294922,
      "logps/rejected": -219.65521240234375,
      "loss": 0.0199,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9030211567878723,
      "rewards/margins": 6.53228759765625,
      "rewards/rejected": -7.435308933258057,
      "step": 1276
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.2104267607743057e-05,
      "logits/chosen": -2.0019478797912598,
      "logits/rejected": -1.9647893905639648,
      "logps/chosen": -228.04661560058594,
      "logps/rejected": -289.8867492675781,
      "loss": 0.0895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6534480452537537,
      "rewards/margins": 7.074681758880615,
      "rewards/rejected": -7.728128910064697,
      "step": 1277
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.2068681554083345e-05,
      "logits/chosen": -1.2781230211257935,
      "logits/rejected": -1.3584738969802856,
      "logps/chosen": -173.55386352539062,
      "logps/rejected": -240.93519592285156,
      "loss": 0.0389,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4009463787078857,
      "rewards/margins": 5.920157432556152,
      "rewards/rejected": -7.321104526519775,
      "step": 1278
    },
    {
      "epoch": 1.67,
      "learning_rate": 2.2033101521603113e-05,
      "logits/chosen": -1.9011565446853638,
      "logits/rejected": -1.9371837377548218,
      "logps/chosen": -149.66355895996094,
      "logps/rejected": -223.8424072265625,
      "loss": 0.0673,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5833083391189575,
      "rewards/margins": 7.0730366706848145,
      "rewards/rejected": -7.656345367431641,
      "step": 1279
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.199752758338679e-05,
      "logits/chosen": -1.9136122465133667,
      "logits/rejected": -1.9128668308258057,
      "logps/chosen": -149.05438232421875,
      "logps/rejected": -208.98980712890625,
      "loss": 0.1537,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1913771629333496,
      "rewards/margins": 5.498708724975586,
      "rewards/rejected": -6.690086364746094,
      "step": 1280
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.19619598125063e-05,
      "logits/chosen": -1.7356456518173218,
      "logits/rejected": -1.8382734060287476,
      "logps/chosen": -149.90243530273438,
      "logps/rejected": -238.3179931640625,
      "loss": 0.0684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6970015168190002,
      "rewards/margins": 6.8320112228393555,
      "rewards/rejected": -7.529013156890869,
      "step": 1281
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.192639828202089e-05,
      "logits/chosen": -1.8271152973175049,
      "logits/rejected": -1.789305567741394,
      "logps/chosen": -246.36155700683594,
      "logps/rejected": -294.989501953125,
      "loss": 0.0687,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4425239562988281,
      "rewards/margins": 5.625908851623535,
      "rewards/rejected": -6.068432331085205,
      "step": 1282
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.1890843064976986e-05,
      "logits/chosen": -1.9487106800079346,
      "logits/rejected": -1.9730805158615112,
      "logps/chosen": -164.71682739257812,
      "logps/rejected": -257.5311279296875,
      "loss": 0.017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.011706210672855377,
      "rewards/margins": 8.925042152404785,
      "rewards/rejected": -8.936749458312988,
      "step": 1283
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.1855294234408068e-05,
      "logits/chosen": -1.8382275104522705,
      "logits/rejected": -1.837470531463623,
      "logps/chosen": -150.8749237060547,
      "logps/rejected": -206.47763061523438,
      "loss": 0.056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7948940992355347,
      "rewards/margins": 5.811569690704346,
      "rewards/rejected": -6.606463432312012,
      "step": 1284
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.181975186333448e-05,
      "logits/chosen": -1.8364896774291992,
      "logits/rejected": -1.868506908416748,
      "logps/chosen": -141.32933044433594,
      "logps/rejected": -198.59872436523438,
      "loss": 0.0717,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5116745233535767,
      "rewards/margins": 5.8407111167907715,
      "rewards/rejected": -6.352385520935059,
      "step": 1285
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.1784216024763284e-05,
      "logits/chosen": -1.828039526939392,
      "logits/rejected": -1.9087718725204468,
      "logps/chosen": -156.95578002929688,
      "logps/rejected": -255.37991333007812,
      "loss": 0.0351,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2033638954162598,
      "rewards/margins": 7.44338846206665,
      "rewards/rejected": -8.646751403808594,
      "step": 1286
    },
    {
      "epoch": 1.68,
      "learning_rate": 2.1748686791688176e-05,
      "logits/chosen": -1.931117296218872,
      "logits/rejected": -1.9398255348205566,
      "logps/chosen": -194.45127868652344,
      "logps/rejected": -259.8490295410156,
      "loss": 0.0645,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0411779880523682,
      "rewards/margins": 6.28025484085083,
      "rewards/rejected": -7.321433067321777,
      "step": 1287
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.1713164237089203e-05,
      "logits/chosen": -1.9279643297195435,
      "logits/rejected": -2.0217397212982178,
      "logps/chosen": -140.97227478027344,
      "logps/rejected": -215.4667510986328,
      "loss": 0.1221,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2633918523788452,
      "rewards/margins": 6.216027736663818,
      "rewards/rejected": -7.479419708251953,
      "step": 1288
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.167764843393277e-05,
      "logits/chosen": -2.0814085006713867,
      "logits/rejected": -2.1000680923461914,
      "logps/chosen": -152.82632446289062,
      "logps/rejected": -223.95718383789062,
      "loss": 0.114,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.29730674624443054,
      "rewards/margins": 6.081691265106201,
      "rewards/rejected": -6.378997802734375,
      "step": 1289
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.1642139455171366e-05,
      "logits/chosen": -2.017010450363159,
      "logits/rejected": -2.0397374629974365,
      "logps/chosen": -159.70016479492188,
      "logps/rejected": -197.58441162109375,
      "loss": 0.1568,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.205440640449524,
      "rewards/margins": 4.722214221954346,
      "rewards/rejected": -5.9276556968688965,
      "step": 1290
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.160663737374348e-05,
      "logits/chosen": -1.7695180177688599,
      "logits/rejected": -1.7639265060424805,
      "logps/chosen": -256.4208984375,
      "logps/rejected": -311.5037841796875,
      "loss": 0.1937,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4590312242507935,
      "rewards/margins": 6.160267353057861,
      "rewards/rejected": -7.619298934936523,
      "step": 1291
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.1571142262573457e-05,
      "logits/chosen": -1.7976187467575073,
      "logits/rejected": -1.7850797176361084,
      "logps/chosen": -147.22457885742188,
      "logps/rejected": -208.04876708984375,
      "loss": 0.0542,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.48498284816741943,
      "rewards/margins": 6.012218952178955,
      "rewards/rejected": -6.497200965881348,
      "step": 1292
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.153565419457126e-05,
      "logits/chosen": -1.7477754354476929,
      "logits/rejected": -1.7613751888275146,
      "logps/chosen": -197.337890625,
      "logps/rejected": -255.48080444335938,
      "loss": 0.1264,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.575129747390747,
      "rewards/margins": 5.721743583679199,
      "rewards/rejected": -7.296873569488525,
      "step": 1293
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.1500173242632446e-05,
      "logits/chosen": -1.9217727184295654,
      "logits/rejected": -1.9335038661956787,
      "logps/chosen": -184.05145263671875,
      "logps/rejected": -238.00938415527344,
      "loss": 0.2433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5255014896392822,
      "rewards/margins": 6.998639106750488,
      "rewards/rejected": -7.52414083480835,
      "step": 1294
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.1464699479637934e-05,
      "logits/chosen": -1.9926395416259766,
      "logits/rejected": -2.0040183067321777,
      "logps/chosen": -134.4732666015625,
      "logps/rejected": -200.7257537841797,
      "loss": 0.0784,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6696519255638123,
      "rewards/margins": 6.800880432128906,
      "rewards/rejected": -7.4705328941345215,
      "step": 1295
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.1429232978453862e-05,
      "logits/chosen": -2.014955759048462,
      "logits/rejected": -2.0637474060058594,
      "logps/chosen": -186.13572692871094,
      "logps/rejected": -272.557861328125,
      "loss": 0.0668,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0366578102111816,
      "rewards/margins": 7.511784553527832,
      "rewards/rejected": -8.548442840576172,
      "step": 1296
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.1393773811931483e-05,
      "logits/chosen": -1.8714430332183838,
      "logits/rejected": -1.91987144947052,
      "logps/chosen": -154.92498779296875,
      "logps/rejected": -241.65760803222656,
      "loss": 0.0409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4182801246643066,
      "rewards/margins": 7.312973499298096,
      "rewards/rejected": -8.731253623962402,
      "step": 1297
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.135832205290696e-05,
      "logits/chosen": -1.9614441394805908,
      "logits/rejected": -1.9798409938812256,
      "logps/chosen": -172.28216552734375,
      "logps/rejected": -234.90675354003906,
      "loss": 0.0916,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6813265085220337,
      "rewards/margins": 6.709239482879639,
      "rewards/rejected": -7.390566349029541,
      "step": 1298
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.132287777420124e-05,
      "logits/chosen": -1.8128993511199951,
      "logits/rejected": -1.7756061553955078,
      "logps/chosen": -172.66470336914062,
      "logps/rejected": -192.32858276367188,
      "loss": 0.0971,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.16137957572937,
      "rewards/margins": 3.952378988265991,
      "rewards/rejected": -6.113758563995361,
      "step": 1299
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.128744104861991e-05,
      "logits/chosen": -1.9525847434997559,
      "logits/rejected": -1.9284476041793823,
      "logps/chosen": -169.3585662841797,
      "logps/rejected": -247.97006225585938,
      "loss": 0.0145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9221512079238892,
      "rewards/margins": 7.623265266418457,
      "rewards/rejected": -8.545416831970215,
      "step": 1300
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.125201194895305e-05,
      "logits/chosen": -1.6681206226348877,
      "logits/rejected": -1.7243953943252563,
      "logps/chosen": -174.7191162109375,
      "logps/rejected": -245.7591552734375,
      "loss": 0.0736,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9889529347419739,
      "rewards/margins": 6.263525009155273,
      "rewards/rejected": -7.25247859954834,
      "step": 1301
    },
    {
      "epoch": 1.7,
      "learning_rate": 2.121659054797507e-05,
      "logits/chosen": -1.965946078300476,
      "logits/rejected": -1.9413881301879883,
      "logps/chosen": -163.7813720703125,
      "logps/rejected": -222.29296875,
      "loss": 0.0285,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.49077728390693665,
      "rewards/margins": 6.804999828338623,
      "rewards/rejected": -7.295776844024658,
      "step": 1302
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.118117691844456e-05,
      "logits/chosen": -1.7857215404510498,
      "logits/rejected": -1.7695603370666504,
      "logps/chosen": -165.377197265625,
      "logps/rejected": -233.5136260986328,
      "loss": 0.0588,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6307117938995361,
      "rewards/margins": 6.847200870513916,
      "rewards/rejected": -7.477912902832031,
      "step": 1303
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.1145771133104157e-05,
      "logits/chosen": -2.03432297706604,
      "logits/rejected": -2.0121405124664307,
      "logps/chosen": -155.52407836914062,
      "logps/rejected": -222.66244506835938,
      "loss": 0.0338,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.038861632347107,
      "rewards/margins": 6.102921485900879,
      "rewards/rejected": -7.141783714294434,
      "step": 1304
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.111037326468037e-05,
      "logits/chosen": -1.8053746223449707,
      "logits/rejected": -1.8246692419052124,
      "logps/chosen": -174.55667114257812,
      "logps/rejected": -244.82728576660156,
      "loss": 0.0558,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6864604949951172,
      "rewards/margins": 6.198990821838379,
      "rewards/rejected": -6.885451316833496,
      "step": 1305
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.107498338588347e-05,
      "logits/chosen": -1.8988919258117676,
      "logits/rejected": -1.830007791519165,
      "logps/chosen": -156.62017822265625,
      "logps/rejected": -203.31298828125,
      "loss": 0.1035,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.14598125219345093,
      "rewards/margins": 6.506600856781006,
      "rewards/rejected": -6.652582168579102,
      "step": 1306
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.1039601569407298e-05,
      "logits/chosen": -1.7457695007324219,
      "logits/rejected": -1.6762486696243286,
      "logps/chosen": -162.04812622070312,
      "logps/rejected": -258.86083984375,
      "loss": 0.0746,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2016370296478271,
      "rewards/margins": 7.707097053527832,
      "rewards/rejected": -8.908734321594238,
      "step": 1307
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.1004227887929133e-05,
      "logits/chosen": -1.862761378288269,
      "logits/rejected": -1.9312154054641724,
      "logps/chosen": -152.9835205078125,
      "logps/rejected": -208.40347290039062,
      "loss": 0.0909,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.952690064907074,
      "rewards/margins": 5.532845497131348,
      "rewards/rejected": -6.485535144805908,
      "step": 1308
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.0968862414109567e-05,
      "logits/chosen": -1.8953279256820679,
      "logits/rejected": -1.8317538499832153,
      "logps/chosen": -170.49267578125,
      "logps/rejected": -245.45118713378906,
      "loss": 0.0052,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.18349838256835938,
      "rewards/margins": 8.552760124206543,
      "rewards/rejected": -8.736259460449219,
      "step": 1309
    },
    {
      "epoch": 1.71,
      "learning_rate": 2.0933505220592295e-05,
      "logits/chosen": -1.8734161853790283,
      "logits/rejected": -1.8214983940124512,
      "logps/chosen": -167.9659423828125,
      "logps/rejected": -233.32077026367188,
      "loss": 0.0539,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9049456715583801,
      "rewards/margins": 5.9225945472717285,
      "rewards/rejected": -6.827540397644043,
      "step": 1310
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.0898156380004034e-05,
      "logits/chosen": -1.997857928276062,
      "logits/rejected": -1.9958791732788086,
      "logps/chosen": -181.889892578125,
      "logps/rejected": -241.04737854003906,
      "loss": 0.127,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.983637809753418,
      "rewards/margins": 5.663478851318359,
      "rewards/rejected": -6.647116184234619,
      "step": 1311
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.086281596495434e-05,
      "logits/chosen": -1.5091166496276855,
      "logits/rejected": -1.4946080446243286,
      "logps/chosen": -153.02407836914062,
      "logps/rejected": -241.63088989257812,
      "loss": 0.0483,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.767093300819397,
      "rewards/margins": 7.854908466339111,
      "rewards/rejected": -8.622000694274902,
      "step": 1312
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.0827484048035445e-05,
      "logits/chosen": -2.075680732727051,
      "logits/rejected": -2.0108556747436523,
      "logps/chosen": -181.0638885498047,
      "logps/rejected": -201.10549926757812,
      "loss": 0.1057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7954965829849243,
      "rewards/margins": 5.348484516143799,
      "rewards/rejected": -6.143980979919434,
      "step": 1313
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.0792160701822157e-05,
      "logits/chosen": -1.8698104619979858,
      "logits/rejected": -1.913940668106079,
      "logps/chosen": -199.4500274658203,
      "logps/rejected": -267.60626220703125,
      "loss": 0.0514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7963769435882568,
      "rewards/margins": 7.418877124786377,
      "rewards/rejected": -8.215253829956055,
      "step": 1314
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.0756845998871623e-05,
      "logits/chosen": -2.1148087978363037,
      "logits/rejected": -2.169675827026367,
      "logps/chosen": -178.546630859375,
      "logps/rejected": -223.95570373535156,
      "loss": 0.166,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2571635246276855,
      "rewards/margins": 5.772641181945801,
      "rewards/rejected": -7.029804229736328,
      "step": 1315
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.07215400117233e-05,
      "logits/chosen": -1.737926959991455,
      "logits/rejected": -1.7428876161575317,
      "logps/chosen": -165.88125610351562,
      "logps/rejected": -253.49676513671875,
      "loss": 0.0877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15423105657100677,
      "rewards/margins": 8.387068748474121,
      "rewards/rejected": -8.541299819946289,
      "step": 1316
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.068624281289871e-05,
      "logits/chosen": -1.8868681192398071,
      "logits/rejected": -1.8737717866897583,
      "logps/chosen": -181.2873077392578,
      "logps/rejected": -244.51266479492188,
      "loss": 0.0181,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.200083389878273,
      "rewards/margins": 7.411710739135742,
      "rewards/rejected": -7.611794471740723,
      "step": 1317
    },
    {
      "epoch": 1.72,
      "learning_rate": 2.065095447490131e-05,
      "logits/chosen": -1.7584081888198853,
      "logits/rejected": -1.8038580417633057,
      "logps/chosen": -174.97018432617188,
      "logps/rejected": -213.44134521484375,
      "loss": 0.1492,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5073225498199463,
      "rewards/margins": 4.993060111999512,
      "rewards/rejected": -6.500383377075195,
      "step": 1318
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.0615675070216393e-05,
      "logits/chosen": -1.7549370527267456,
      "logits/rejected": -1.7759162187576294,
      "logps/chosen": -191.2140655517578,
      "logps/rejected": -248.96768188476562,
      "loss": 0.0607,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.9590156078338623,
      "rewards/margins": 4.876646041870117,
      "rewards/rejected": -7.835661888122559,
      "step": 1319
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.0580404671310878e-05,
      "logits/chosen": -1.6139053106307983,
      "logits/rejected": -1.5718433856964111,
      "logps/chosen": -160.9483184814453,
      "logps/rejected": -196.6767578125,
      "loss": 0.0378,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.006493091583252,
      "rewards/margins": 4.882761478424072,
      "rewards/rejected": -5.889255046844482,
      "step": 1320
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.0545143350633177e-05,
      "logits/chosen": -1.7707104682922363,
      "logits/rejected": -1.7290619611740112,
      "logps/chosen": -159.78370666503906,
      "logps/rejected": -247.87200927734375,
      "loss": 0.0914,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12345397472381592,
      "rewards/margins": 8.418564796447754,
      "rewards/rejected": -8.295110702514648,
      "step": 1321
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.0509891180613066e-05,
      "logits/chosen": -1.8615283966064453,
      "logits/rejected": -1.8557054996490479,
      "logps/chosen": -165.07801818847656,
      "logps/rejected": -212.31802368164062,
      "loss": 0.0638,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0801455974578857,
      "rewards/margins": 5.734646797180176,
      "rewards/rejected": -6.814792156219482,
      "step": 1322
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.0474648233661543e-05,
      "logits/chosen": -2.037238121032715,
      "logits/rejected": -2.0281758308410645,
      "logps/chosen": -169.9398193359375,
      "logps/rejected": -233.53758239746094,
      "loss": 0.061,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6495194435119629,
      "rewards/margins": 7.2947797775268555,
      "rewards/rejected": -7.94429874420166,
      "step": 1323
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.0439414582170628e-05,
      "logits/chosen": -1.967191219329834,
      "logits/rejected": -2.0180180072784424,
      "logps/chosen": -162.86770629882812,
      "logps/rejected": -226.91661071777344,
      "loss": 0.1422,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1102142333984375,
      "rewards/margins": 6.198751449584961,
      "rewards/rejected": -7.308966636657715,
      "step": 1324
    },
    {
      "epoch": 1.73,
      "learning_rate": 2.040419029851328e-05,
      "logits/chosen": -1.875981092453003,
      "logits/rejected": -1.9078865051269531,
      "logps/chosen": -155.73838806152344,
      "logps/rejected": -226.39651489257812,
      "loss": 0.04,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9254744052886963,
      "rewards/margins": 7.150867462158203,
      "rewards/rejected": -8.07634162902832,
      "step": 1325
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.0368975455043178e-05,
      "logits/chosen": -2.0002667903900146,
      "logits/rejected": -2.004883050918579,
      "logps/chosen": -157.9558563232422,
      "logps/rejected": -200.30892944335938,
      "loss": 0.1337,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6380078196525574,
      "rewards/margins": 4.977096080780029,
      "rewards/rejected": -5.6151041984558105,
      "step": 1326
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.033377012409463e-05,
      "logits/chosen": -1.989454984664917,
      "logits/rejected": -2.058359384536743,
      "logps/chosen": -133.8167266845703,
      "logps/rejected": -199.6424560546875,
      "loss": 0.1341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8248870372772217,
      "rewards/margins": 5.525320529937744,
      "rewards/rejected": -6.350208282470703,
      "step": 1327
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.0298574377982427e-05,
      "logits/chosen": -1.9729851484298706,
      "logits/rejected": -2.0309276580810547,
      "logps/chosen": -156.47897338867188,
      "logps/rejected": -212.2986602783203,
      "loss": 0.0838,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5218062400817871,
      "rewards/margins": 5.931440353393555,
      "rewards/rejected": -6.453246593475342,
      "step": 1328
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.026338828900163e-05,
      "logits/chosen": -1.9390101432800293,
      "logits/rejected": -1.9235492944717407,
      "logps/chosen": -160.933837890625,
      "logps/rejected": -217.399169921875,
      "loss": 0.0403,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.08333158493042,
      "rewards/margins": 6.369418621063232,
      "rewards/rejected": -7.452749729156494,
      "step": 1329
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.022821192942749e-05,
      "logits/chosen": -1.965376853942871,
      "logits/rejected": -1.997107744216919,
      "logps/chosen": -162.6165313720703,
      "logps/rejected": -224.1361846923828,
      "loss": 0.0417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0327022075653076,
      "rewards/margins": 6.00932502746582,
      "rewards/rejected": -8.04202651977539,
      "step": 1330
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.0193045371515276e-05,
      "logits/chosen": -1.8546010255813599,
      "logits/rejected": -1.784804344177246,
      "logps/chosen": -182.34898376464844,
      "logps/rejected": -258.6595764160156,
      "loss": 0.1296,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9826211333274841,
      "rewards/margins": 7.850172996520996,
      "rewards/rejected": -8.832793235778809,
      "step": 1331
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.015788868750009e-05,
      "logits/chosen": -1.8911024332046509,
      "logits/rejected": -1.9000606536865234,
      "logps/chosen": -172.73020935058594,
      "logps/rejected": -251.44642639160156,
      "loss": 0.0061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.674170970916748,
      "rewards/margins": 7.54857873916626,
      "rewards/rejected": -8.222750663757324,
      "step": 1332
    },
    {
      "epoch": 1.74,
      "learning_rate": 2.0122741949596797e-05,
      "logits/chosen": -1.8977237939834595,
      "logits/rejected": -1.875288486480713,
      "logps/chosen": -167.87098693847656,
      "logps/rejected": -250.7166748046875,
      "loss": 0.0624,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02346806228160858,
      "rewards/margins": 8.797714233398438,
      "rewards/rejected": -8.774246215820312,
      "step": 1333
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.00876052299998e-05,
      "logits/chosen": -2.0416619777679443,
      "logits/rejected": -1.927350640296936,
      "logps/chosen": -176.74224853515625,
      "logps/rejected": -249.3499298095703,
      "loss": 0.0089,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0490537881851196,
      "rewards/margins": 7.213248252868652,
      "rewards/rejected": -8.26230239868164,
      "step": 1334
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.0052478600882935e-05,
      "logits/chosen": -1.5342981815338135,
      "logits/rejected": -1.5252878665924072,
      "logps/chosen": -140.84475708007812,
      "logps/rejected": -200.86424255371094,
      "loss": 0.2055,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9733503460884094,
      "rewards/margins": 4.943357467651367,
      "rewards/rejected": -5.916707515716553,
      "step": 1335
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.001736213439933e-05,
      "logits/chosen": -1.8892353773117065,
      "logits/rejected": -1.9409749507904053,
      "logps/chosen": -263.30029296875,
      "logps/rejected": -362.54901123046875,
      "loss": 0.1534,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9487963914871216,
      "rewards/margins": 8.060709953308105,
      "rewards/rejected": -9.009506225585938,
      "step": 1336
    },
    {
      "epoch": 1.75,
      "learning_rate": 1.9982255902681186e-05,
      "logits/chosen": -1.7874882221221924,
      "logits/rejected": -1.8309708833694458,
      "logps/chosen": -143.11639404296875,
      "logps/rejected": -214.7310791015625,
      "loss": 0.0482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.007793493568897247,
      "rewards/margins": 7.3343610763549805,
      "rewards/rejected": -7.342154502868652,
      "step": 1337
    },
    {
      "epoch": 1.75,
      "learning_rate": 1.9947159977839736e-05,
      "logits/chosen": -1.5530506372451782,
      "logits/rejected": -1.533398151397705,
      "logps/chosen": -177.9195556640625,
      "logps/rejected": -265.0281982421875,
      "loss": 0.0712,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.532582402229309,
      "rewards/margins": 7.494189262390137,
      "rewards/rejected": -9.026772499084473,
      "step": 1338
    },
    {
      "epoch": 1.75,
      "learning_rate": 1.991207443196501e-05,
      "logits/chosen": -1.7862417697906494,
      "logits/rejected": -1.830432415008545,
      "logps/chosen": -161.42514038085938,
      "logps/rejected": -231.53887939453125,
      "loss": 0.0261,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6619853973388672,
      "rewards/margins": 7.097233772277832,
      "rewards/rejected": -7.759219169616699,
      "step": 1339
    },
    {
      "epoch": 1.75,
      "learning_rate": 1.987699933712573e-05,
      "logits/chosen": -1.925607442855835,
      "logits/rejected": -1.8962010145187378,
      "logps/chosen": -156.5908203125,
      "logps/rejected": -195.99734497070312,
      "loss": 0.0876,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6970105171203613,
      "rewards/margins": 5.735270977020264,
      "rewards/rejected": -6.432281970977783,
      "step": 1340
    },
    {
      "epoch": 1.75,
      "learning_rate": 1.9841934765369153e-05,
      "logits/chosen": -1.8474020957946777,
      "logits/rejected": -1.8351906538009644,
      "logps/chosen": -141.69830322265625,
      "logps/rejected": -215.289306640625,
      "loss": 0.0686,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.524068295955658,
      "rewards/margins": 7.474521636962891,
      "rewards/rejected": -7.998589515686035,
      "step": 1341
    },
    {
      "epoch": 1.76,
      "learning_rate": 1.9806880788720916e-05,
      "logits/chosen": -1.998929738998413,
      "logits/rejected": -1.9390027523040771,
      "logps/chosen": -163.60069274902344,
      "logps/rejected": -219.84619140625,
      "loss": 0.1036,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7701488733291626,
      "rewards/margins": 6.155312538146973,
      "rewards/rejected": -7.925462245941162,
      "step": 1342
    },
    {
      "epoch": 1.76,
      "learning_rate": 1.977183747918489e-05,
      "logits/chosen": -1.9086220264434814,
      "logits/rejected": -1.9184178113937378,
      "logps/chosen": -192.9392547607422,
      "logps/rejected": -250.08152770996094,
      "loss": 0.1788,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4920424222946167,
      "rewards/margins": 5.28303861618042,
      "rewards/rejected": -6.775080680847168,
      "step": 1343
    },
    {
      "epoch": 1.76,
      "learning_rate": 1.9736804908743033e-05,
      "logits/chosen": -1.7996941804885864,
      "logits/rejected": -1.7998764514923096,
      "logps/chosen": -172.49078369140625,
      "logps/rejected": -224.74154663085938,
      "loss": 0.2154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1966731548309326,
      "rewards/margins": 4.981950759887695,
      "rewards/rejected": -6.178623676300049,
      "step": 1344
    },
    {
      "epoch": 1.76,
      "learning_rate": 1.9701783149355255e-05,
      "logits/chosen": -2.0108978748321533,
      "logits/rejected": -2.057300090789795,
      "logps/chosen": -168.12060546875,
      "logps/rejected": -213.21209716796875,
      "loss": 0.1483,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.816092848777771,
      "rewards/margins": 5.075588703155518,
      "rewards/rejected": -5.891682147979736,
      "step": 1345
    },
    {
      "epoch": 1.76,
      "learning_rate": 1.9666772272959253e-05,
      "logits/chosen": -2.0527470111846924,
      "logits/rejected": -2.024061679840088,
      "logps/chosen": -169.88430786132812,
      "logps/rejected": -242.3800048828125,
      "loss": 0.0767,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6212195158004761,
      "rewards/margins": 7.578874111175537,
      "rewards/rejected": -8.200094223022461,
      "step": 1346
    },
    {
      "epoch": 1.76,
      "learning_rate": 1.9631772351470383e-05,
      "logits/chosen": -1.7058093547821045,
      "logits/rejected": -1.7809479236602783,
      "logps/chosen": -174.14047241210938,
      "logps/rejected": -286.26531982421875,
      "loss": 0.0232,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6088874936103821,
      "rewards/margins": 7.1679277420043945,
      "rewards/rejected": -7.776814937591553,
      "step": 1347
    },
    {
      "epoch": 1.76,
      "learning_rate": 1.959678345678146e-05,
      "logits/chosen": -1.8513472080230713,
      "logits/rejected": -1.7793071269989014,
      "logps/chosen": -262.7961120605469,
      "logps/rejected": -349.125244140625,
      "loss": 0.0602,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5631558895111084,
      "rewards/margins": 7.166615009307861,
      "rewards/rejected": -8.72977066040039,
      "step": 1348
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9561805660762684e-05,
      "logits/chosen": -1.8268765211105347,
      "logits/rejected": -1.8081550598144531,
      "logps/chosen": -205.76824951171875,
      "logps/rejected": -285.45086669921875,
      "loss": 0.1414,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2240908294916153,
      "rewards/margins": 7.87926721572876,
      "rewards/rejected": -8.103357315063477,
      "step": 1349
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.952683903526145e-05,
      "logits/chosen": -2.0640740394592285,
      "logits/rejected": -2.0389645099639893,
      "logps/chosen": -142.57398986816406,
      "logps/rejected": -204.6869354248047,
      "loss": 0.1863,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.017685897648334503,
      "rewards/margins": 5.801783084869385,
      "rewards/rejected": -5.819469451904297,
      "step": 1350
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9491883652102208e-05,
      "logits/chosen": -1.7569046020507812,
      "logits/rejected": -1.8102633953094482,
      "logps/chosen": -181.20654296875,
      "logps/rejected": -247.11248779296875,
      "loss": 0.0115,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.010845378041267395,
      "rewards/margins": 7.3789753913879395,
      "rewards/rejected": -7.3898210525512695,
      "step": 1351
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9456939583086303e-05,
      "logits/chosen": -2.008316993713379,
      "logits/rejected": -2.0166409015655518,
      "logps/chosen": -167.07260131835938,
      "logps/rejected": -237.5175018310547,
      "loss": 0.1599,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4066956043243408,
      "rewards/margins": 5.0600175857543945,
      "rewards/rejected": -6.466713905334473,
      "step": 1352
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9422006899991878e-05,
      "logits/chosen": -1.7467762231826782,
      "logits/rejected": -1.738730549812317,
      "logps/chosen": -171.87841796875,
      "logps/rejected": -232.4315643310547,
      "loss": 0.1546,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.385115146636963,
      "rewards/margins": 5.692521095275879,
      "rewards/rejected": -7.077637195587158,
      "step": 1353
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9387085674573616e-05,
      "logits/chosen": -1.7682383060455322,
      "logits/rejected": -1.7932897806167603,
      "logps/chosen": -145.12530517578125,
      "logps/rejected": -193.86961364746094,
      "loss": 0.1553,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1851158142089844,
      "rewards/margins": 4.217765808105469,
      "rewards/rejected": -5.402882099151611,
      "step": 1354
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9352175978562736e-05,
      "logits/chosen": -2.0004708766937256,
      "logits/rejected": -1.9849834442138672,
      "logps/chosen": -155.1116485595703,
      "logps/rejected": -224.90625,
      "loss": 0.0533,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7847661972045898,
      "rewards/margins": 6.646977424621582,
      "rewards/rejected": -7.4317426681518555,
      "step": 1355
    },
    {
      "epoch": 1.77,
      "learning_rate": 1.9317277883666745e-05,
      "logits/chosen": -1.9150443077087402,
      "logits/rejected": -1.9050016403198242,
      "logps/chosen": -199.38218688964844,
      "logps/rejected": -255.24745178222656,
      "loss": 0.1735,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8390464782714844,
      "rewards/margins": 5.411545753479004,
      "rewards/rejected": -6.250593185424805,
      "step": 1356
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.9282391461569316e-05,
      "logits/chosen": -1.8502684831619263,
      "logits/rejected": -1.8841150999069214,
      "logps/chosen": -165.98912048339844,
      "logps/rejected": -235.4638671875,
      "loss": 0.1286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2726131677627563,
      "rewards/margins": 6.386384963989258,
      "rewards/rejected": -7.658998012542725,
      "step": 1357
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.924751678393017e-05,
      "logits/chosen": -1.5814719200134277,
      "logits/rejected": -1.5452768802642822,
      "logps/chosen": -155.57078552246094,
      "logps/rejected": -214.4939727783203,
      "loss": 0.1019,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.626254141330719,
      "rewards/margins": 6.881619453430176,
      "rewards/rejected": -7.50787353515625,
      "step": 1358
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.9212653922384854e-05,
      "logits/chosen": -1.871513843536377,
      "logits/rejected": -1.8207221031188965,
      "logps/chosen": -199.4781951904297,
      "logps/rejected": -241.53822326660156,
      "loss": 0.2026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3943023681640625,
      "rewards/margins": 5.372564315795898,
      "rewards/rejected": -6.766867637634277,
      "step": 1359
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.91778029485447e-05,
      "logits/chosen": -1.8576443195343018,
      "logits/rejected": -1.899790644645691,
      "logps/chosen": -165.27774047851562,
      "logps/rejected": -232.46554565429688,
      "loss": 0.0371,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8429927229881287,
      "rewards/margins": 7.583266258239746,
      "rewards/rejected": -8.42625904083252,
      "step": 1360
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.914296393399659e-05,
      "logits/chosen": -2.0466842651367188,
      "logits/rejected": -2.0269927978515625,
      "logps/chosen": -206.87326049804688,
      "logps/rejected": -244.22195434570312,
      "loss": 0.0653,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7498255372047424,
      "rewards/margins": 6.343250751495361,
      "rewards/rejected": -7.093076229095459,
      "step": 1361
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.910813695030284e-05,
      "logits/chosen": -1.4926820993423462,
      "logits/rejected": -1.5543553829193115,
      "logps/chosen": -125.51263427734375,
      "logps/rejected": -206.45501708984375,
      "loss": 0.1484,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6498977541923523,
      "rewards/margins": 6.6911301612854,
      "rewards/rejected": -7.341028213500977,
      "step": 1362
    },
    {
      "epoch": 1.78,
      "learning_rate": 1.9073322069001075e-05,
      "logits/chosen": -1.9525163173675537,
      "logits/rejected": -1.9201407432556152,
      "logps/chosen": -160.992431640625,
      "logps/rejected": -226.40518188476562,
      "loss": 0.0752,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8022050857543945,
      "rewards/margins": 6.84550666809082,
      "rewards/rejected": -7.647711753845215,
      "step": 1363
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.9038519361604046e-05,
      "logits/chosen": -1.8327667713165283,
      "logits/rejected": -1.9223092794418335,
      "logps/chosen": -137.26817321777344,
      "logps/rejected": -219.43971252441406,
      "loss": 0.152,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9201211929321289,
      "rewards/margins": 6.84136438369751,
      "rewards/rejected": -7.7614850997924805,
      "step": 1364
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.900372889959949e-05,
      "logits/chosen": -1.896927833557129,
      "logits/rejected": -1.8377330303192139,
      "logps/chosen": -167.00059509277344,
      "logps/rejected": -239.67849731445312,
      "loss": 0.0882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7887258529663086,
      "rewards/margins": 7.188932418823242,
      "rewards/rejected": -7.977658271789551,
      "step": 1365
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.896895075445e-05,
      "logits/chosen": -1.7759754657745361,
      "logits/rejected": -1.7599891424179077,
      "logps/chosen": -220.1333465576172,
      "logps/rejected": -318.3359069824219,
      "loss": 0.0593,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2547904253005981,
      "rewards/margins": 8.493195533752441,
      "rewards/rejected": -9.747986793518066,
      "step": 1366
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.8934184997592866e-05,
      "logits/chosen": -1.7002334594726562,
      "logits/rejected": -1.7340943813323975,
      "logps/chosen": -149.20669555664062,
      "logps/rejected": -223.23873901367188,
      "loss": 0.1107,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7146911025047302,
      "rewards/margins": 7.014072418212891,
      "rewards/rejected": -7.728763580322266,
      "step": 1367
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.8899431700439946e-05,
      "logits/chosen": -1.6825141906738281,
      "logits/rejected": -1.6885271072387695,
      "logps/chosen": -170.6098175048828,
      "logps/rejected": -288.748779296875,
      "loss": 0.1058,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3645206391811371,
      "rewards/margins": 7.580642223358154,
      "rewards/rejected": -7.945162773132324,
      "step": 1368
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.8864690934377492e-05,
      "logits/chosen": -1.92576003074646,
      "logits/rejected": -1.904494047164917,
      "logps/chosen": -160.45425415039062,
      "logps/rejected": -233.24868774414062,
      "loss": 0.0404,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0287816524505615,
      "rewards/margins": 7.097235679626465,
      "rewards/rejected": -8.126016616821289,
      "step": 1369
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.8829962770766003e-05,
      "logits/chosen": -1.976628065109253,
      "logits/rejected": -1.9786465167999268,
      "logps/chosen": -177.93411254882812,
      "logps/rejected": -251.54052734375,
      "loss": 0.0525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2664155960083008,
      "rewards/margins": 7.834298133850098,
      "rewards/rejected": -8.100713729858398,
      "step": 1370
    },
    {
      "epoch": 1.79,
      "learning_rate": 1.8795247280940108e-05,
      "logits/chosen": -1.528140902519226,
      "logits/rejected": -1.5092068910598755,
      "logps/chosen": -188.1293487548828,
      "logps/rejected": -249.56573486328125,
      "loss": 0.0142,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.194149136543274,
      "rewards/margins": 6.973577499389648,
      "rewards/rejected": -8.16772747039795,
      "step": 1371
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.876054453620841e-05,
      "logits/chosen": -1.8395880460739136,
      "logits/rejected": -1.8692125082015991,
      "logps/chosen": -157.14117431640625,
      "logps/rejected": -223.36978149414062,
      "loss": 0.0533,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8454115986824036,
      "rewards/margins": 6.862707614898682,
      "rewards/rejected": -7.7081193923950195,
      "step": 1372
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.872585460785332e-05,
      "logits/chosen": -1.9672234058380127,
      "logits/rejected": -1.950184941291809,
      "logps/chosen": -165.4678955078125,
      "logps/rejected": -233.02554321289062,
      "loss": 0.0211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6089394092559814,
      "rewards/margins": 7.490427017211914,
      "rewards/rejected": -8.099367141723633,
      "step": 1373
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.869117756713092e-05,
      "logits/chosen": -2.137336492538452,
      "logits/rejected": -2.099393367767334,
      "logps/chosen": -145.5584716796875,
      "logps/rejected": -207.6409454345703,
      "loss": 0.1328,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.257631540298462,
      "rewards/margins": 4.999992847442627,
      "rewards/rejected": -6.25762414932251,
      "step": 1374
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.8656513485270843e-05,
      "logits/chosen": -1.8002862930297852,
      "logits/rejected": -1.767629861831665,
      "logps/chosen": -142.00326538085938,
      "logps/rejected": -214.62063598632812,
      "loss": 0.0173,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9868193864822388,
      "rewards/margins": 7.139993667602539,
      "rewards/rejected": -8.126812934875488,
      "step": 1375
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.8621862433476054e-05,
      "logits/chosen": -1.9125896692276,
      "logits/rejected": -1.9144890308380127,
      "logps/chosen": -289.9126892089844,
      "logps/rejected": -347.1576232910156,
      "loss": 0.1619,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0552219152450562,
      "rewards/margins": 5.4380059242248535,
      "rewards/rejected": -6.493227481842041,
      "step": 1376
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.858722448292281e-05,
      "logits/chosen": -1.8929309844970703,
      "logits/rejected": -1.9591344594955444,
      "logps/chosen": -168.857666015625,
      "logps/rejected": -232.99493408203125,
      "loss": 0.1528,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7905786633491516,
      "rewards/margins": 5.6970343589782715,
      "rewards/rejected": -6.487612724304199,
      "step": 1377
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.8552599704760424e-05,
      "logits/chosen": -1.9652599096298218,
      "logits/rejected": -1.9844290018081665,
      "logps/chosen": -164.49395751953125,
      "logps/rejected": -245.54652404785156,
      "loss": 0.0161,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.41764047741889954,
      "rewards/margins": 6.947850227355957,
      "rewards/rejected": -7.365490436553955,
      "step": 1378
    },
    {
      "epoch": 1.8,
      "learning_rate": 1.851798817011116e-05,
      "logits/chosen": -1.7243952751159668,
      "logits/rejected": -1.7436703443527222,
      "logps/chosen": -129.95407104492188,
      "logps/rejected": -197.84410095214844,
      "loss": 0.2039,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0999517440795898,
      "rewards/margins": 5.47808837890625,
      "rewards/rejected": -6.57804012298584,
      "step": 1379
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.8483389950070097e-05,
      "logits/chosen": -1.9925131797790527,
      "logits/rejected": -2.0441794395446777,
      "logps/chosen": -134.2703857421875,
      "logps/rejected": -178.04290771484375,
      "loss": 0.1154,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4518166780471802,
      "rewards/margins": 5.368523597717285,
      "rewards/rejected": -5.820339202880859,
      "step": 1380
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.8448805115704903e-05,
      "logits/chosen": -1.5169048309326172,
      "logits/rejected": -1.5278306007385254,
      "logps/chosen": -157.79425048828125,
      "logps/rejected": -219.03765869140625,
      "loss": 0.1708,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.18181566894054413,
      "rewards/margins": 7.183194160461426,
      "rewards/rejected": -7.3650102615356445,
      "step": 1381
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.841423373805583e-05,
      "logits/chosen": -2.002209424972534,
      "logits/rejected": -2.0072743892669678,
      "logps/chosen": -154.25296020507812,
      "logps/rejected": -231.89739990234375,
      "loss": 0.0434,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1144790649414062,
      "rewards/margins": 6.965666770935059,
      "rewards/rejected": -8.080145835876465,
      "step": 1382
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.837967588813544e-05,
      "logits/chosen": -1.838489055633545,
      "logits/rejected": -1.777534008026123,
      "logps/chosen": -158.78456115722656,
      "logps/rejected": -209.16445922851562,
      "loss": 0.063,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.42065542936325073,
      "rewards/margins": 6.696077823638916,
      "rewards/rejected": -7.116732597351074,
      "step": 1383
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.8345131636928518e-05,
      "logits/chosen": -1.9571161270141602,
      "logits/rejected": -1.9555604457855225,
      "logps/chosen": -217.0253143310547,
      "logps/rejected": -282.6556091308594,
      "loss": 0.0959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8585553765296936,
      "rewards/margins": 6.7290873527526855,
      "rewards/rejected": -7.587642192840576,
      "step": 1384
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.8310601055391923e-05,
      "logits/chosen": -2.09067964553833,
      "logits/rejected": -2.1356773376464844,
      "logps/chosen": -165.89022827148438,
      "logps/rejected": -246.5386199951172,
      "loss": 0.0132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5034182667732239,
      "rewards/margins": 7.330573081970215,
      "rewards/rejected": -7.833991050720215,
      "step": 1385
    },
    {
      "epoch": 1.81,
      "learning_rate": 1.8276084214454443e-05,
      "logits/chosen": -1.54730224609375,
      "logits/rejected": -1.5326613187789917,
      "logps/chosen": -173.9701690673828,
      "logps/rejected": -258.54412841796875,
      "loss": 0.0554,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4913601279258728,
      "rewards/margins": 8.67663288116455,
      "rewards/rejected": -9.16799259185791,
      "step": 1386
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.8241581185016603e-05,
      "logits/chosen": -1.8564624786376953,
      "logits/rejected": -1.8772978782653809,
      "logps/chosen": -126.70140075683594,
      "logps/rejected": -204.8530731201172,
      "loss": 0.0377,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5302541851997375,
      "rewards/margins": 6.721182823181152,
      "rewards/rejected": -7.251437187194824,
      "step": 1387
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.8207092037950602e-05,
      "logits/chosen": -1.980771541595459,
      "logits/rejected": -1.980690360069275,
      "logps/chosen": -158.46725463867188,
      "logps/rejected": -229.17428588867188,
      "loss": 0.0915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7634891271591187,
      "rewards/margins": 6.673912048339844,
      "rewards/rejected": -7.437402248382568,
      "step": 1388
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.8172616844100096e-05,
      "logits/chosen": -1.7929273843765259,
      "logits/rejected": -1.8288214206695557,
      "logps/chosen": -156.375,
      "logps/rejected": -201.7799530029297,
      "loss": 0.0466,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7693904638290405,
      "rewards/margins": 4.838718414306641,
      "rewards/rejected": -5.608108997344971,
      "step": 1389
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.81381556742801e-05,
      "logits/chosen": -1.8647431135177612,
      "logits/rejected": -1.8619341850280762,
      "logps/chosen": -163.07762145996094,
      "logps/rejected": -231.95347595214844,
      "loss": 0.0815,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2728267908096313,
      "rewards/margins": 6.834033489227295,
      "rewards/rejected": -8.106860160827637,
      "step": 1390
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.8103708599276812e-05,
      "logits/chosen": -1.9335469007492065,
      "logits/rejected": -2.046647787094116,
      "logps/chosen": -180.75889587402344,
      "logps/rejected": -254.34140014648438,
      "loss": 0.1064,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.087051510810852,
      "rewards/margins": 6.8290534019470215,
      "rewards/rejected": -7.916104793548584,
      "step": 1391
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.8069275689847466e-05,
      "logits/chosen": -1.7985718250274658,
      "logits/rejected": -1.8040627241134644,
      "logps/chosen": -181.0277099609375,
      "logps/rejected": -216.98861694335938,
      "loss": 0.1429,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6740210652351379,
      "rewards/margins": 6.335559368133545,
      "rewards/rejected": -7.009579658508301,
      "step": 1392
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.803485701672022e-05,
      "logits/chosen": -1.8536020517349243,
      "logits/rejected": -1.8877567052841187,
      "logps/chosen": -199.85287475585938,
      "logps/rejected": -251.81996154785156,
      "loss": 0.1404,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3715848922729492,
      "rewards/margins": 4.707953453063965,
      "rewards/rejected": -6.079538345336914,
      "step": 1393
    },
    {
      "epoch": 1.82,
      "learning_rate": 1.8000452650593976e-05,
      "logits/chosen": -1.9024213552474976,
      "logits/rejected": -1.961869716644287,
      "logps/chosen": -166.1037139892578,
      "logps/rejected": -237.1409912109375,
      "loss": 0.05,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.44713306427001953,
      "rewards/margins": 6.956056594848633,
      "rewards/rejected": -7.403189182281494,
      "step": 1394
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.7966062662138262e-05,
      "logits/chosen": -1.863601565361023,
      "logits/rejected": -1.8587448596954346,
      "logps/chosen": -180.49658203125,
      "logps/rejected": -255.48934936523438,
      "loss": 0.0211,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7160875797271729,
      "rewards/margins": 6.545836448669434,
      "rewards/rejected": -8.261923789978027,
      "step": 1395
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.7931687121993047e-05,
      "logits/chosen": -2.0440375804901123,
      "logits/rejected": -2.063211679458618,
      "logps/chosen": -217.48512268066406,
      "logps/rejected": -310.9078063964844,
      "loss": 0.1741,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9756447672843933,
      "rewards/margins": 7.42236852645874,
      "rewards/rejected": -8.3980131149292,
      "step": 1396
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.7897326100768664e-05,
      "logits/chosen": -1.9256500005722046,
      "logits/rejected": -1.9821003675460815,
      "logps/chosen": -135.53097534179688,
      "logps/rejected": -225.1648712158203,
      "loss": 0.2121,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5473628044128418,
      "rewards/margins": 5.493593692779541,
      "rewards/rejected": -6.040956497192383,
      "step": 1397
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.7862979669045566e-05,
      "logits/chosen": -1.9225335121154785,
      "logits/rejected": -1.9265072345733643,
      "logps/chosen": -154.5044708251953,
      "logps/rejected": -230.73483276367188,
      "loss": 0.0502,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8898203372955322,
      "rewards/margins": 7.631102561950684,
      "rewards/rejected": -8.520922660827637,
      "step": 1398
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.782864789737429e-05,
      "logits/chosen": -1.749634861946106,
      "logits/rejected": -1.771803617477417,
      "logps/chosen": -173.75558471679688,
      "logps/rejected": -229.7580108642578,
      "loss": 0.0672,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19316892325878143,
      "rewards/margins": 7.1682047843933105,
      "rewards/rejected": -7.3613739013671875,
      "step": 1399
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.779433085627523e-05,
      "logits/chosen": -2.054530620574951,
      "logits/rejected": -2.0558972358703613,
      "logps/chosen": -154.94308471679688,
      "logps/rejected": -189.52117919921875,
      "loss": 0.2093,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.066015362739563,
      "rewards/margins": 4.387103080749512,
      "rewards/rejected": -5.453118324279785,
      "step": 1400
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.7760028616238535e-05,
      "logits/chosen": -1.6870012283325195,
      "logits/rejected": -1.7101577520370483,
      "logps/chosen": -136.96688842773438,
      "logps/rejected": -231.81710815429688,
      "loss": 0.04,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7918201684951782,
      "rewards/margins": 8.164579391479492,
      "rewards/rejected": -8.956398963928223,
      "step": 1401
    },
    {
      "epoch": 1.83,
      "learning_rate": 1.7725741247723965e-05,
      "logits/chosen": -1.8054518699645996,
      "logits/rejected": -1.7903985977172852,
      "logps/chosen": -173.0701141357422,
      "logps/rejected": -257.371337890625,
      "loss": 0.0178,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.2992980480194092,
      "rewards/margins": 7.613722324371338,
      "rewards/rejected": -7.913020610809326,
      "step": 1402
    },
    {
      "epoch": 1.84,
      "learning_rate": 1.769146882116068e-05,
      "logits/chosen": -1.896955966949463,
      "logits/rejected": -1.8798946142196655,
      "logps/chosen": -136.32186889648438,
      "logps/rejected": -199.8182830810547,
      "loss": 0.1366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.0095314979553223,
      "rewards/margins": 6.072952747344971,
      "rewards/rejected": -7.082484245300293,
      "step": 1403
    },
    {
      "epoch": 1.84,
      "learning_rate": 1.7657211406947206e-05,
      "logits/chosen": -1.725835919380188,
      "logits/rejected": -1.7542049884796143,
      "logps/chosen": -160.03887939453125,
      "logps/rejected": -251.65786743164062,
      "loss": 0.0854,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6318047046661377,
      "rewards/margins": 8.690157890319824,
      "rewards/rejected": -9.3219633102417,
      "step": 1404
    },
    {
      "epoch": 1.84,
      "learning_rate": 1.7622969075451204e-05,
      "logits/chosen": -2.127474546432495,
      "logits/rejected": -2.1183981895446777,
      "logps/chosen": -181.05438232421875,
      "logps/rejected": -228.97178649902344,
      "loss": 0.0505,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2711164951324463,
      "rewards/margins": 7.137664318084717,
      "rewards/rejected": -7.408781051635742,
      "step": 1405
    },
    {
      "epoch": 1.84,
      "learning_rate": 1.758874189700936e-05,
      "logits/chosen": -1.9130439758300781,
      "logits/rejected": -1.811869502067566,
      "logps/chosen": -160.6462860107422,
      "logps/rejected": -206.53575134277344,
      "loss": 0.1524,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.347366213798523,
      "rewards/margins": 6.096449375152588,
      "rewards/rejected": -7.443815231323242,
      "step": 1406
    },
    {
      "epoch": 1.84,
      "learning_rate": 1.7554529941927243e-05,
      "logits/chosen": -1.9717810153961182,
      "logits/rejected": -1.9884655475616455,
      "logps/chosen": -231.16017150878906,
      "logps/rejected": -302.2288818359375,
      "loss": 0.1707,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6486893892288208,
      "rewards/margins": 5.758804798126221,
      "rewards/rejected": -7.40749454498291,
      "step": 1407
    },
    {
      "epoch": 1.84,
      "learning_rate": 1.7520333280479124e-05,
      "logits/chosen": -1.9157713651657104,
      "logits/rejected": -1.9877333641052246,
      "logps/chosen": -200.7000274658203,
      "logps/rejected": -283.57763671875,
      "loss": 0.1665,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8140336871147156,
      "rewards/margins": 6.561330795288086,
      "rewards/rejected": -7.375364303588867,
      "step": 1408
    },
    {
      "epoch": 1.84,
      "learning_rate": 1.7486151982907896e-05,
      "logits/chosen": -1.9267053604125977,
      "logits/rejected": -1.881373405456543,
      "logps/chosen": -169.57815551757812,
      "logps/rejected": -233.28660583496094,
      "loss": 0.0393,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7489716410636902,
      "rewards/margins": 5.724695205688477,
      "rewards/rejected": -6.473666667938232,
      "step": 1409
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.7451986119424863e-05,
      "logits/chosen": -1.7978085279464722,
      "logits/rejected": -1.800220251083374,
      "logps/chosen": -157.89239501953125,
      "logps/rejected": -255.89186096191406,
      "loss": 0.223,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.898529589176178,
      "rewards/margins": 6.3009467124938965,
      "rewards/rejected": -7.19947624206543,
      "step": 1410
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.7417835760209638e-05,
      "logits/chosen": -1.7216917276382446,
      "logits/rejected": -1.7697508335113525,
      "logps/chosen": -140.20799255371094,
      "logps/rejected": -197.74798583984375,
      "loss": 0.1266,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5246366262435913,
      "rewards/margins": 6.1821417808532715,
      "rewards/rejected": -6.706778526306152,
      "step": 1411
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.738370097541e-05,
      "logits/chosen": -1.9050142765045166,
      "logits/rejected": -1.8743025064468384,
      "logps/chosen": -182.64474487304688,
      "logps/rejected": -225.1285400390625,
      "loss": 0.0542,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8847573399543762,
      "rewards/margins": 6.631954193115234,
      "rewards/rejected": -7.5167107582092285,
      "step": 1412
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.7349581835141725e-05,
      "logits/chosen": -1.5119991302490234,
      "logits/rejected": -1.5632303953170776,
      "logps/chosen": -168.77044677734375,
      "logps/rejected": -234.53286743164062,
      "loss": 0.0708,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.047499388456344604,
      "rewards/margins": 7.054121017456055,
      "rewards/rejected": -7.101619720458984,
      "step": 1413
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.7315478409488436e-05,
      "logits/chosen": -1.7494206428527832,
      "logits/rejected": -1.725780725479126,
      "logps/chosen": -151.7712860107422,
      "logps/rejected": -190.1285400390625,
      "loss": 0.1719,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3345469236373901,
      "rewards/margins": 4.226208209991455,
      "rewards/rejected": -5.560755252838135,
      "step": 1414
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.72813907685015e-05,
      "logits/chosen": -1.9111223220825195,
      "logits/rejected": -1.958429217338562,
      "logps/chosen": -167.26766967773438,
      "logps/rejected": -215.09576416015625,
      "loss": 0.2392,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.6266288757324219,
      "rewards/margins": 6.916955471038818,
      "rewards/rejected": -7.543583869934082,
      "step": 1415
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.7247318982199862e-05,
      "logits/chosen": -1.8830338716506958,
      "logits/rejected": -1.9052337408065796,
      "logps/chosen": -154.9111785888672,
      "logps/rejected": -218.30526733398438,
      "loss": 0.0862,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1239248588681221,
      "rewards/margins": 7.576015949249268,
      "rewards/rejected": -7.452091217041016,
      "step": 1416
    },
    {
      "epoch": 1.85,
      "learning_rate": 1.721326312056989e-05,
      "logits/chosen": -1.489595651626587,
      "logits/rejected": -1.5095163583755493,
      "logps/chosen": -185.0439453125,
      "logps/rejected": -255.82948303222656,
      "loss": 0.147,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.4467023611068726,
      "rewards/margins": 6.324016571044922,
      "rewards/rejected": -7.770719528198242,
      "step": 1417
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.717922325356525e-05,
      "logits/chosen": -1.9401211738586426,
      "logits/rejected": -1.9825904369354248,
      "logps/chosen": -154.74542236328125,
      "logps/rejected": -225.6487579345703,
      "loss": 0.1547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.29825735092163086,
      "rewards/margins": 6.03098201751709,
      "rewards/rejected": -6.3292388916015625,
      "step": 1418
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.7145199451106736e-05,
      "logits/chosen": -1.8931381702423096,
      "logits/rejected": -1.9087814092636108,
      "logps/chosen": -201.10986328125,
      "logps/rejected": -268.57373046875,
      "loss": 0.0385,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4573646783828735,
      "rewards/margins": 6.188492298126221,
      "rewards/rejected": -7.645857334136963,
      "step": 1419
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.7111191783082155e-05,
      "logits/chosen": -1.7465510368347168,
      "logits/rejected": -1.8003541231155396,
      "logps/chosen": -177.22860717773438,
      "logps/rejected": -259.50665283203125,
      "loss": 0.0184,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7660298347473145,
      "rewards/margins": 7.652481555938721,
      "rewards/rejected": -8.418512344360352,
      "step": 1420
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.7077200319346186e-05,
      "logits/chosen": -1.8556287288665771,
      "logits/rejected": -1.903159499168396,
      "logps/chosen": -159.71458435058594,
      "logps/rejected": -253.72320556640625,
      "loss": 0.0511,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.413325309753418,
      "rewards/margins": 6.68924617767334,
      "rewards/rejected": -8.102571487426758,
      "step": 1421
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.7043225129720207e-05,
      "logits/chosen": -1.6158943176269531,
      "logits/rejected": -1.6229677200317383,
      "logps/chosen": -170.93655395507812,
      "logps/rejected": -217.23233032226562,
      "loss": 0.0458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.542958676815033,
      "rewards/margins": 6.423084259033203,
      "rewards/rejected": -6.966043472290039,
      "step": 1422
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.7009266283992163e-05,
      "logits/chosen": -1.744412899017334,
      "logits/rejected": -1.7253538370132446,
      "logps/chosen": -159.83575439453125,
      "logps/rejected": -217.30685424804688,
      "loss": 0.0871,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0497480630874634,
      "rewards/margins": 5.708693504333496,
      "rewards/rejected": -6.758441925048828,
      "step": 1423
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.6975323851916454e-05,
      "logits/chosen": -1.6443414688110352,
      "logits/rejected": -1.6907676458358765,
      "logps/chosen": -172.95944213867188,
      "logps/rejected": -248.3804931640625,
      "loss": 0.0538,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3981698751449585,
      "rewards/margins": 6.256303787231445,
      "rewards/rejected": -7.654473781585693,
      "step": 1424
    },
    {
      "epoch": 1.86,
      "learning_rate": 1.6941397903213717e-05,
      "logits/chosen": -1.6515917778015137,
      "logits/rejected": -1.5929831266403198,
      "logps/chosen": -176.04031372070312,
      "logps/rejected": -250.02015686035156,
      "loss": 0.0716,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3851500153541565,
      "rewards/margins": 7.349740982055664,
      "rewards/rejected": -7.734891414642334,
      "step": 1425
    },
    {
      "epoch": 1.87,
      "learning_rate": 1.6907488507570786e-05,
      "logits/chosen": -1.7157336473464966,
      "logits/rejected": -1.7122169733047485,
      "logps/chosen": -178.92269897460938,
      "logps/rejected": -244.966064453125,
      "loss": 0.2034,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6244277954101562,
      "rewards/margins": 6.766473770141602,
      "rewards/rejected": -8.390900611877441,
      "step": 1426
    },
    {
      "epoch": 1.87,
      "learning_rate": 1.6873595734640457e-05,
      "logits/chosen": -1.820783257484436,
      "logits/rejected": -1.901712417602539,
      "logps/chosen": -159.55783081054688,
      "logps/rejected": -193.87908935546875,
      "loss": 0.1934,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5285515785217285,
      "rewards/margins": 4.2751970291137695,
      "rewards/rejected": -5.803748607635498,
      "step": 1427
    },
    {
      "epoch": 1.87,
      "learning_rate": 1.683971965404139e-05,
      "logits/chosen": -1.563103199005127,
      "logits/rejected": -1.6024824380874634,
      "logps/chosen": -184.21389770507812,
      "logps/rejected": -269.5892028808594,
      "loss": 0.0734,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5729142427444458,
      "rewards/margins": 6.669857025146484,
      "rewards/rejected": -8.24277114868164,
      "step": 1428
    },
    {
      "epoch": 1.87,
      "learning_rate": 1.6805860335357977e-05,
      "logits/chosen": -1.9113903045654297,
      "logits/rejected": -1.9137660264968872,
      "logps/chosen": -144.79747009277344,
      "logps/rejected": -190.53701782226562,
      "loss": 0.098,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7563795447349548,
      "rewards/margins": 4.923892498016357,
      "rewards/rejected": -5.680271625518799,
      "step": 1429
    },
    {
      "epoch": 1.87,
      "learning_rate": 1.6772017848140132e-05,
      "logits/chosen": -2.051774740219116,
      "logits/rejected": -2.0785787105560303,
      "logps/chosen": -155.75283813476562,
      "logps/rejected": -212.421630859375,
      "loss": 0.0935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3198063373565674,
      "rewards/margins": 6.431396484375,
      "rewards/rejected": -6.751202583312988,
      "step": 1430
    },
    {
      "epoch": 1.87,
      "learning_rate": 1.6738192261903248e-05,
      "logits/chosen": -1.6132410764694214,
      "logits/rejected": -1.6706408262252808,
      "logps/chosen": -184.29200744628906,
      "logps/rejected": -262.5608215332031,
      "loss": 0.0503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6118072271347046,
      "rewards/margins": 7.0644850730896,
      "rewards/rejected": -7.6762919425964355,
      "step": 1431
    },
    {
      "epoch": 1.87,
      "learning_rate": 1.6704383646127973e-05,
      "logits/chosen": -1.929612159729004,
      "logits/rejected": -1.8774811029434204,
      "logps/chosen": -148.33314514160156,
      "logps/rejected": -204.91915893554688,
      "loss": 0.0816,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3227298259735107,
      "rewards/margins": 5.273367881774902,
      "rewards/rejected": -6.596096992492676,
      "step": 1432
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.6670592070260106e-05,
      "logits/chosen": -1.7057464122772217,
      "logits/rejected": -1.7081327438354492,
      "logps/chosen": -167.19642639160156,
      "logps/rejected": -246.28250122070312,
      "loss": 0.0592,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7460168600082397,
      "rewards/margins": 6.99407958984375,
      "rewards/rejected": -7.740096569061279,
      "step": 1433
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.6636817603710437e-05,
      "logits/chosen": -1.9624965190887451,
      "logits/rejected": -1.9680092334747314,
      "logps/chosen": -162.2013397216797,
      "logps/rejected": -217.49496459960938,
      "loss": 0.1182,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8050522804260254,
      "rewards/margins": 5.3024492263793945,
      "rewards/rejected": -7.107501983642578,
      "step": 1434
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.660306031585463e-05,
      "logits/chosen": -1.7863792181015015,
      "logits/rejected": -1.8888020515441895,
      "logps/chosen": -164.04408264160156,
      "logps/rejected": -206.6036376953125,
      "loss": 0.2947,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.0345616340637207,
      "rewards/margins": 4.184384346008301,
      "rewards/rejected": -6.21894645690918,
      "step": 1435
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.6569320276033034e-05,
      "logits/chosen": -2.044656753540039,
      "logits/rejected": -2.048750400543213,
      "logps/chosen": -182.16943359375,
      "logps/rejected": -266.421142578125,
      "loss": 0.1424,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3561677932739258,
      "rewards/margins": 6.173374176025391,
      "rewards/rejected": -7.529542922973633,
      "step": 1436
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.653559755355058e-05,
      "logits/chosen": -1.4633909463882446,
      "logits/rejected": -1.3710346221923828,
      "logps/chosen": -202.04501342773438,
      "logps/rejected": -228.17987060546875,
      "loss": 0.1924,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -1.2592594623565674,
      "rewards/margins": 4.48421573638916,
      "rewards/rejected": -5.743475437164307,
      "step": 1437
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.6501892217676653e-05,
      "logits/chosen": -2.031708240509033,
      "logits/rejected": -2.0355191230773926,
      "logps/chosen": -147.04547119140625,
      "logps/rejected": -219.62109375,
      "loss": 0.0751,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.840111494064331,
      "rewards/margins": 6.610848903656006,
      "rewards/rejected": -7.450960159301758,
      "step": 1438
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.6468204337644887e-05,
      "logits/chosen": -1.9042456150054932,
      "logits/rejected": -1.9146372079849243,
      "logps/chosen": -174.64988708496094,
      "logps/rejected": -242.59503173828125,
      "loss": 0.0586,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7033993601799011,
      "rewards/margins": 6.440855503082275,
      "rewards/rejected": -7.144254684448242,
      "step": 1439
    },
    {
      "epoch": 1.88,
      "learning_rate": 1.643453398265309e-05,
      "logits/chosen": -2.0373077392578125,
      "logits/rejected": -1.9990719556808472,
      "logps/chosen": -197.56239318847656,
      "logps/rejected": -224.86566162109375,
      "loss": 0.1843,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0612456798553467,
      "rewards/margins": 4.7939982414245605,
      "rewards/rejected": -5.855244159698486,
      "step": 1440
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.6400881221863044e-05,
      "logits/chosen": -2.101670026779175,
      "logits/rejected": -2.0808441638946533,
      "logps/chosen": -202.12460327148438,
      "logps/rejected": -270.551025390625,
      "loss": 0.0988,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.879106879234314,
      "rewards/margins": 6.384647846221924,
      "rewards/rejected": -7.2637553215026855,
      "step": 1441
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.6367246124400402e-05,
      "logits/chosen": -1.9559378623962402,
      "logits/rejected": -1.97991144657135,
      "logps/chosen": -138.27597045898438,
      "logps/rejected": -225.91539001464844,
      "loss": 0.0567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0869765281677246,
      "rewards/margins": 7.538537502288818,
      "rewards/rejected": -8.625514030456543,
      "step": 1442
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.633362875935456e-05,
      "logits/chosen": -1.9537116289138794,
      "logits/rejected": -1.9170485734939575,
      "logps/chosen": -170.5803985595703,
      "logps/rejected": -220.02813720703125,
      "loss": 0.133,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5233724117279053,
      "rewards/margins": 6.517301559448242,
      "rewards/rejected": -7.040674209594727,
      "step": 1443
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.6300029195778455e-05,
      "logits/chosen": -1.9791290760040283,
      "logits/rejected": -1.9995908737182617,
      "logps/chosen": -148.0351104736328,
      "logps/rejected": -209.7469940185547,
      "loss": 0.0959,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6104867458343506,
      "rewards/margins": 5.6612772941589355,
      "rewards/rejected": -6.271763801574707,
      "step": 1444
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.626644750268847e-05,
      "logits/chosen": -2.0022377967834473,
      "logits/rejected": -2.04081654548645,
      "logps/chosen": -203.27479553222656,
      "logps/rejected": -271.150390625,
      "loss": 0.1026,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7666787505149841,
      "rewards/margins": 5.506726264953613,
      "rewards/rejected": -6.273406028747559,
      "step": 1445
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.62328837490643e-05,
      "logits/chosen": -2.0479040145874023,
      "logits/rejected": -2.0359928607940674,
      "logps/chosen": -205.13970947265625,
      "logps/rejected": -296.627685546875,
      "loss": 0.0553,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3939322531223297,
      "rewards/margins": 8.30208969116211,
      "rewards/rejected": -8.69602108001709,
      "step": 1446
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.6199338003848745e-05,
      "logits/chosen": -1.754265546798706,
      "logits/rejected": -1.7747364044189453,
      "logps/chosen": -179.20904541015625,
      "logps/rejected": -250.15399169921875,
      "loss": 0.0754,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4154643714427948,
      "rewards/margins": 6.89732027053833,
      "rewards/rejected": -7.312784194946289,
      "step": 1447
    },
    {
      "epoch": 1.89,
      "learning_rate": 1.6165810335947664e-05,
      "logits/chosen": -2.052436351776123,
      "logits/rejected": -2.0595521926879883,
      "logps/chosen": -176.74652099609375,
      "logps/rejected": -205.03985595703125,
      "loss": 0.1509,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2468891739845276,
      "rewards/margins": 4.9088006019592285,
      "rewards/rejected": -5.1556901931762695,
      "step": 1448
    },
    {
      "epoch": 1.9,
      "learning_rate": 1.6132300814229755e-05,
      "logits/chosen": -1.895953893661499,
      "logits/rejected": -1.9909448623657227,
      "logps/chosen": -151.97201538085938,
      "logps/rejected": -224.51316833496094,
      "loss": 0.1315,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2033798098564148,
      "rewards/margins": 6.772775173187256,
      "rewards/rejected": -6.976154327392578,
      "step": 1449
    },
    {
      "epoch": 1.9,
      "learning_rate": 1.6098809507526445e-05,
      "logits/chosen": -1.87679123878479,
      "logits/rejected": -1.9058992862701416,
      "logps/chosen": -223.10098266601562,
      "logps/rejected": -337.18463134765625,
      "loss": 0.1148,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5640072822570801,
      "rewards/margins": 8.61253833770752,
      "rewards/rejected": -9.176546096801758,
      "step": 1450
    },
    {
      "epoch": 1.9,
      "learning_rate": 1.606533648463177e-05,
      "logits/chosen": -1.556974172592163,
      "logits/rejected": -1.584230661392212,
      "logps/chosen": -183.27992248535156,
      "logps/rejected": -276.49163818359375,
      "loss": 0.0291,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4861931800842285,
      "rewards/margins": 6.5030622482299805,
      "rewards/rejected": -7.989253997802734,
      "step": 1451
    },
    {
      "epoch": 1.9,
      "learning_rate": 1.603188181430216e-05,
      "logits/chosen": -1.98918616771698,
      "logits/rejected": -1.99131441116333,
      "logps/chosen": -172.1450958251953,
      "logps/rejected": -242.2749481201172,
      "loss": 0.0986,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.599496603012085,
      "rewards/margins": 6.56987190246582,
      "rewards/rejected": -7.169368743896484,
      "step": 1452
    },
    {
      "epoch": 1.9,
      "learning_rate": 1.5998445565256398e-05,
      "logits/chosen": -1.9819287061691284,
      "logits/rejected": -2.06848406791687,
      "logps/chosen": -153.8423614501953,
      "logps/rejected": -241.15504455566406,
      "loss": 0.0895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3593679964542389,
      "rewards/margins": 6.629528045654297,
      "rewards/rejected": -6.988896369934082,
      "step": 1453
    },
    {
      "epoch": 1.9,
      "learning_rate": 1.59650278061754e-05,
      "logits/chosen": -1.8113974332809448,
      "logits/rejected": -1.7811760902404785,
      "logps/chosen": -180.42601013183594,
      "logps/rejected": -261.30670166015625,
      "loss": 0.0462,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8879536986351013,
      "rewards/margins": 7.591157913208008,
      "rewards/rejected": -8.479111671447754,
      "step": 1454
    },
    {
      "epoch": 1.9,
      "learning_rate": 1.5931628605702102e-05,
      "logits/chosen": -1.9276123046875,
      "logits/rejected": -1.9165130853652954,
      "logps/chosen": -146.40457153320312,
      "logps/rejected": -197.0927734375,
      "loss": 0.1051,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18943612277507782,
      "rewards/margins": 6.442972660064697,
      "rewards/rejected": -6.253536224365234,
      "step": 1455
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.5898248032441336e-05,
      "logits/chosen": -1.9316844940185547,
      "logits/rejected": -1.9472423791885376,
      "logps/chosen": -153.13470458984375,
      "logps/rejected": -211.61044311523438,
      "loss": 0.0663,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2788617610931396,
      "rewards/margins": 5.574979782104492,
      "rewards/rejected": -6.8538408279418945,
      "step": 1456
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.5864886154959673e-05,
      "logits/chosen": -1.9079139232635498,
      "logits/rejected": -1.8762353658676147,
      "logps/chosen": -168.4247589111328,
      "logps/rejected": -237.31785583496094,
      "loss": 0.101,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2764003872871399,
      "rewards/margins": 7.474781513214111,
      "rewards/rejected": -7.7511820793151855,
      "step": 1457
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.5831543041785247e-05,
      "logits/chosen": -1.8867089748382568,
      "logits/rejected": -1.878395915031433,
      "logps/chosen": -203.84320068359375,
      "logps/rejected": -258.5014343261719,
      "loss": 0.2207,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -1.1975269317626953,
      "rewards/margins": 5.611706256866455,
      "rewards/rejected": -6.809232711791992,
      "step": 1458
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.579821876140768e-05,
      "logits/chosen": -2.0417754650115967,
      "logits/rejected": -2.034687042236328,
      "logps/chosen": -179.48194885253906,
      "logps/rejected": -247.83583068847656,
      "loss": 0.0505,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9956956505775452,
      "rewards/margins": 6.859674453735352,
      "rewards/rejected": -7.855370044708252,
      "step": 1459
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.5764913382277903e-05,
      "logits/chosen": -1.4775530099868774,
      "logits/rejected": -1.6557426452636719,
      "logps/chosen": -186.2441864013672,
      "logps/rejected": -253.23587036132812,
      "loss": 0.0627,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10700085759162903,
      "rewards/margins": 7.619004726409912,
      "rewards/rejected": -7.726005554199219,
      "step": 1460
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.5731626972808027e-05,
      "logits/chosen": -1.8130213022232056,
      "logits/rejected": -1.8273311853408813,
      "logps/chosen": -181.8021697998047,
      "logps/rejected": -216.8883514404297,
      "loss": 0.0871,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.745829463005066,
      "rewards/margins": 4.812769889831543,
      "rewards/rejected": -6.558599472045898,
      "step": 1461
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.5698359601371187e-05,
      "logits/chosen": -2.0159366130828857,
      "logits/rejected": -2.125441789627075,
      "logps/chosen": -165.08578491210938,
      "logps/rejected": -235.95504760742188,
      "loss": 0.0503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0819928646087646,
      "rewards/margins": 6.00016450881958,
      "rewards/rejected": -7.082157135009766,
      "step": 1462
    },
    {
      "epoch": 1.91,
      "learning_rate": 1.5665111336301415e-05,
      "logits/chosen": -1.8478294610977173,
      "logits/rejected": -1.8737025260925293,
      "logps/chosen": -167.20901489257812,
      "logps/rejected": -291.16619873046875,
      "loss": 0.0323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2196100950241089,
      "rewards/margins": 8.42543888092041,
      "rewards/rejected": -8.205829620361328,
      "step": 1463
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.563188224589349e-05,
      "logits/chosen": -1.897483468055725,
      "logits/rejected": -1.8279160261154175,
      "logps/chosen": -155.93907165527344,
      "logps/rejected": -204.47634887695312,
      "loss": 0.1037,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0618826150894165,
      "rewards/margins": 5.23366641998291,
      "rewards/rejected": -6.295549392700195,
      "step": 1464
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.5598672398402835e-05,
      "logits/chosen": -2.0118744373321533,
      "logits/rejected": -1.9417507648468018,
      "logps/chosen": -154.65557861328125,
      "logps/rejected": -226.17263793945312,
      "loss": 0.0533,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7694597840309143,
      "rewards/margins": 6.348083019256592,
      "rewards/rejected": -7.117542266845703,
      "step": 1465
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.5565481862045312e-05,
      "logits/chosen": -1.636594533920288,
      "logits/rejected": -1.6213279962539673,
      "logps/chosen": -157.98915100097656,
      "logps/rejected": -234.44046020507812,
      "loss": 0.0147,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22608618438243866,
      "rewards/margins": 7.148587226867676,
      "rewards/rejected": -7.374673366546631,
      "step": 1466
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.553231070499712e-05,
      "logits/chosen": -2.0592000484466553,
      "logits/rejected": -2.099379777908325,
      "logps/chosen": -154.08773803710938,
      "logps/rejected": -220.42359924316406,
      "loss": 0.0872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6953530311584473,
      "rewards/margins": 5.255290508270264,
      "rewards/rejected": -6.950644016265869,
      "step": 1467
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.549915899539469e-05,
      "logits/chosen": -1.9446617364883423,
      "logits/rejected": -1.9726697206497192,
      "logps/chosen": -161.54420471191406,
      "logps/rejected": -213.55789184570312,
      "loss": 0.2078,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.73211669921875,
      "rewards/margins": 4.925564765930176,
      "rewards/rejected": -5.657681465148926,
      "step": 1468
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.5466026801334437e-05,
      "logits/chosen": -1.6446067094802856,
      "logits/rejected": -1.6120967864990234,
      "logps/chosen": -133.76028442382812,
      "logps/rejected": -194.2477264404297,
      "loss": 0.0672,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.27902817726135254,
      "rewards/margins": 6.187336444854736,
      "rewards/rejected": -6.466365337371826,
      "step": 1469
    },
    {
      "epoch": 1.92,
      "learning_rate": 1.5432914190872757e-05,
      "logits/chosen": -2.1161065101623535,
      "logits/rejected": -2.1727135181427,
      "logps/chosen": -151.29400634765625,
      "logps/rejected": -218.5394744873047,
      "loss": 0.0564,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.47885987162590027,
      "rewards/margins": 5.689356327056885,
      "rewards/rejected": -6.168215751647949,
      "step": 1470
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.5399821232025786e-05,
      "logits/chosen": -2.072629928588867,
      "logits/rejected": -2.070687770843506,
      "logps/chosen": -277.82421875,
      "logps/rejected": -349.20233154296875,
      "loss": 0.1021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.152440071105957,
      "rewards/margins": 6.434161186218262,
      "rewards/rejected": -7.586602210998535,
      "step": 1471
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.5366747992769287e-05,
      "logits/chosen": -1.7929476499557495,
      "logits/rejected": -1.8582980632781982,
      "logps/chosen": -184.43136596679688,
      "logps/rejected": -260.0470275878906,
      "loss": 0.0665,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.469385027885437,
      "rewards/margins": 6.053367614746094,
      "rewards/rejected": -7.522752285003662,
      "step": 1472
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.5333694541038557e-05,
      "logits/chosen": -1.9221478700637817,
      "logits/rejected": -1.9227879047393799,
      "logps/chosen": -180.98336791992188,
      "logps/rejected": -232.79702758789062,
      "loss": 0.066,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1610603332519531,
      "rewards/margins": 5.251230716705322,
      "rewards/rejected": -6.412291526794434,
      "step": 1473
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.5300660944728187e-05,
      "logits/chosen": -1.8955940008163452,
      "logits/rejected": -1.862095832824707,
      "logps/chosen": -153.5137939453125,
      "logps/rejected": -202.85977172851562,
      "loss": 0.138,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9767177104949951,
      "rewards/margins": 5.11998987197876,
      "rewards/rejected": -6.096707344055176,
      "step": 1474
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.5267647271692036e-05,
      "logits/chosen": -1.9131454229354858,
      "logits/rejected": -1.884170413017273,
      "logps/chosen": -175.6882781982422,
      "logps/rejected": -238.2341766357422,
      "loss": 0.0253,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7169787883758545,
      "rewards/margins": 7.063757419586182,
      "rewards/rejected": -7.780735969543457,
      "step": 1475
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.523465358974302e-05,
      "logits/chosen": -2.053736686706543,
      "logits/rejected": -2.1396591663360596,
      "logps/chosen": -168.14321899414062,
      "logps/rejected": -269.98565673828125,
      "loss": 0.0482,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.40043923258781433,
      "rewards/margins": 8.295912742614746,
      "rewards/rejected": -8.6963529586792,
      "step": 1476
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.5201679966652981e-05,
      "logits/chosen": -1.8836390972137451,
      "logits/rejected": -1.8183432817459106,
      "logps/chosen": -151.0546875,
      "logps/rejected": -222.21046447753906,
      "loss": 0.0651,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.329729676246643,
      "rewards/margins": 6.630373001098633,
      "rewards/rejected": -7.960102558135986,
      "step": 1477
    },
    {
      "epoch": 1.93,
      "learning_rate": 1.5168726470152583e-05,
      "logits/chosen": -1.7073020935058594,
      "logits/rejected": -1.8206056356430054,
      "logps/chosen": -156.07029724121094,
      "logps/rejected": -243.04547119140625,
      "loss": 0.09,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4592230319976807,
      "rewards/margins": 7.087145805358887,
      "rewards/rejected": -8.546368598937988,
      "step": 1478
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.5135793167931128e-05,
      "logits/chosen": -2.0303549766540527,
      "logits/rejected": -2.0749616622924805,
      "logps/chosen": -161.1666259765625,
      "logps/rejected": -232.07015991210938,
      "loss": 0.0898,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23262013494968414,
      "rewards/margins": 7.038035869598389,
      "rewards/rejected": -6.805416107177734,
      "step": 1479
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.5102880127636438e-05,
      "logits/chosen": -1.7862125635147095,
      "logits/rejected": -1.7472658157348633,
      "logps/chosen": -135.6455841064453,
      "logps/rejected": -179.91275024414062,
      "loss": 0.1683,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8419880270957947,
      "rewards/margins": 4.57899808883667,
      "rewards/rejected": -5.420985698699951,
      "step": 1480
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.506998741687472e-05,
      "logits/chosen": -1.848314881324768,
      "logits/rejected": -1.9159537553787231,
      "logps/chosen": -126.62445831298828,
      "logps/rejected": -200.40264892578125,
      "loss": 0.1248,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7802714109420776,
      "rewards/margins": 5.813082218170166,
      "rewards/rejected": -6.593353748321533,
      "step": 1481
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.5037115103210419e-05,
      "logits/chosen": -1.8367905616760254,
      "logits/rejected": -1.9133474826812744,
      "logps/chosen": -145.2078399658203,
      "logps/rejected": -222.38601684570312,
      "loss": 0.0353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6482294797897339,
      "rewards/margins": 6.907781600952148,
      "rewards/rejected": -7.556010723114014,
      "step": 1482
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.5004263254166107e-05,
      "logits/chosen": -2.1106410026550293,
      "logits/rejected": -2.120593309402466,
      "logps/chosen": -147.8441162109375,
      "logps/rejected": -223.29095458984375,
      "loss": 0.0827,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6013448238372803,
      "rewards/margins": 6.433332443237305,
      "rewards/rejected": -7.034677505493164,
      "step": 1483
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.4971431937222283e-05,
      "logits/chosen": -1.9918663501739502,
      "logits/rejected": -1.9960469007492065,
      "logps/chosen": -144.62586975097656,
      "logps/rejected": -187.34616088867188,
      "loss": 0.1363,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7511521577835083,
      "rewards/margins": 5.059243202209473,
      "rewards/rejected": -6.810395240783691,
      "step": 1484
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.493862121981729e-05,
      "logits/chosen": -1.9147193431854248,
      "logits/rejected": -1.875861644744873,
      "logps/chosen": -162.79713439941406,
      "logps/rejected": -252.04232788085938,
      "loss": 0.0241,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.716283917427063,
      "rewards/margins": 8.625290870666504,
      "rewards/rejected": -9.341573715209961,
      "step": 1485
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.4905831169347145e-05,
      "logits/chosen": -1.861860990524292,
      "logits/rejected": -1.857662320137024,
      "logps/chosen": -174.1083221435547,
      "logps/rejected": -231.5437469482422,
      "loss": 0.1313,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4532400369644165,
      "rewards/margins": 7.360616683959961,
      "rewards/rejected": -7.813856601715088,
      "step": 1486
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.4873061853165444e-05,
      "logits/chosen": -1.6813139915466309,
      "logits/rejected": -1.6601676940917969,
      "logps/chosen": -154.69216918945312,
      "logps/rejected": -238.66831970214844,
      "loss": 0.032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.582141637802124,
      "rewards/margins": 8.182817459106445,
      "rewards/rejected": -8.764959335327148,
      "step": 1487
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.4840313338583162e-05,
      "logits/chosen": -1.9887897968292236,
      "logits/rejected": -2.038120985031128,
      "logps/chosen": -159.79080200195312,
      "logps/rejected": -245.13204956054688,
      "loss": 0.0654,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5657658576965332,
      "rewards/margins": 8.364178657531738,
      "rewards/rejected": -8.92994499206543,
      "step": 1488
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.4807585692868552e-05,
      "logits/chosen": -1.775769829750061,
      "logits/rejected": -1.8334171772003174,
      "logps/chosen": -180.20826721191406,
      "logps/rejected": -253.2466278076172,
      "loss": 0.1426,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8748674392700195,
      "rewards/margins": 6.331639289855957,
      "rewards/rejected": -7.206506252288818,
      "step": 1489
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.4774878983247026e-05,
      "logits/chosen": -1.8238931894302368,
      "logits/rejected": -1.8703408241271973,
      "logps/chosen": -171.43856811523438,
      "logps/rejected": -239.18411254882812,
      "loss": 0.1092,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.259733200073242,
      "rewards/margins": 5.788527488708496,
      "rewards/rejected": -8.048261642456055,
      "step": 1490
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.4742193276900937e-05,
      "logits/chosen": -1.9635274410247803,
      "logits/rejected": -1.9715871810913086,
      "logps/chosen": -202.4301300048828,
      "logps/rejected": -261.9853820800781,
      "loss": 0.0721,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.242929458618164,
      "rewards/margins": 7.041566848754883,
      "rewards/rejected": -8.284496307373047,
      "step": 1491
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.4709528640969552e-05,
      "logits/chosen": -2.0981409549713135,
      "logits/rejected": -2.0858359336853027,
      "logps/chosen": -171.84991455078125,
      "logps/rejected": -248.1218719482422,
      "loss": 0.1192,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6634970903396606,
      "rewards/margins": 7.763343334197998,
      "rewards/rejected": -9.426839828491211,
      "step": 1492
    },
    {
      "epoch": 1.95,
      "learning_rate": 1.4676885142548829e-05,
      "logits/chosen": -1.770486831665039,
      "logits/rejected": -1.7089924812316895,
      "logps/chosen": -185.61412048339844,
      "logps/rejected": -236.9242401123047,
      "loss": 0.1397,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.0511775016784668,
      "rewards/margins": 6.004816055297852,
      "rewards/rejected": -7.05599308013916,
      "step": 1493
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.4644262848691311e-05,
      "logits/chosen": -1.8018348217010498,
      "logits/rejected": -1.8627887964248657,
      "logps/chosen": -158.44735717773438,
      "logps/rejected": -236.19366455078125,
      "loss": 0.0605,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5780305862426758,
      "rewards/margins": 6.049410820007324,
      "rewards/rejected": -7.62744140625,
      "step": 1494
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.4611661826406004e-05,
      "logits/chosen": -1.798191785812378,
      "logits/rejected": -1.7754775285720825,
      "logps/chosen": -173.0909881591797,
      "logps/rejected": -218.6238555908203,
      "loss": 0.0353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7564226388931274,
      "rewards/margins": 6.3637285232543945,
      "rewards/rejected": -8.12015151977539,
      "step": 1495
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.4579082142658176e-05,
      "logits/chosen": -2.0703423023223877,
      "logits/rejected": -2.0821683406829834,
      "logps/chosen": -197.2122802734375,
      "logps/rejected": -279.0089416503906,
      "loss": 0.1475,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5636324882507324,
      "rewards/margins": 6.981237411499023,
      "rewards/rejected": -8.544869422912598,
      "step": 1496
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.4546523864369303e-05,
      "logits/chosen": -1.7798852920532227,
      "logits/rejected": -1.784767985343933,
      "logps/chosen": -146.83432006835938,
      "logps/rejected": -222.25306701660156,
      "loss": 0.0968,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6043685674667358,
      "rewards/margins": 6.432318687438965,
      "rewards/rejected": -8.036687850952148,
      "step": 1497
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.4513987058416879e-05,
      "logits/chosen": -2.0713613033294678,
      "logits/rejected": -2.130263328552246,
      "logps/chosen": -158.7416229248047,
      "logps/rejected": -251.99969482421875,
      "loss": 0.1098,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6425489187240601,
      "rewards/margins": 8.164459228515625,
      "rewards/rejected": -8.8070068359375,
      "step": 1498
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.448147179163431e-05,
      "logits/chosen": -1.9446178674697876,
      "logits/rejected": -2.0181891918182373,
      "logps/chosen": -156.156494140625,
      "logps/rejected": -245.68138122558594,
      "loss": 0.061,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0737344026565552,
      "rewards/margins": 6.920964241027832,
      "rewards/rejected": -7.9946980476379395,
      "step": 1499
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.4448978130810715e-05,
      "logits/chosen": -1.8688730001449585,
      "logits/rejected": -2.004462957382202,
      "logps/chosen": -162.00047302246094,
      "logps/rejected": -269.91632080078125,
      "loss": 0.0902,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8149991035461426,
      "rewards/margins": 8.04463005065918,
      "rewards/rejected": -9.85962963104248,
      "step": 1500
    },
    {
      "epoch": 1.96,
      "learning_rate": 1.4416506142690889e-05,
      "logits/chosen": -1.7957065105438232,
      "logits/rejected": -1.8496233224868774,
      "logps/chosen": -188.8843231201172,
      "logps/rejected": -226.80471801757812,
      "loss": 0.3021,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -2.0921332836151123,
      "rewards/margins": 4.176154136657715,
      "rewards/rejected": -6.268287181854248,
      "step": 1501
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.4384055893975051e-05,
      "logits/chosen": -2.0541560649871826,
      "logits/rejected": -2.044050693511963,
      "logps/chosen": -146.43299865722656,
      "logps/rejected": -194.96112060546875,
      "loss": 0.1513,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.067152738571167,
      "rewards/margins": 5.489653587341309,
      "rewards/rejected": -6.5568060874938965,
      "step": 1502
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.4351627451318821e-05,
      "logits/chosen": -2.068666458129883,
      "logits/rejected": -2.060936689376831,
      "logps/chosen": -177.52618408203125,
      "logps/rejected": -268.50677490234375,
      "loss": 0.0571,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9761327505111694,
      "rewards/margins": 7.710536003112793,
      "rewards/rejected": -8.68666934967041,
      "step": 1503
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.4319220881332979e-05,
      "logits/chosen": -1.6705870628356934,
      "logits/rejected": -1.6689965724945068,
      "logps/chosen": -166.10577392578125,
      "logps/rejected": -220.07373046875,
      "loss": 0.2133,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8065643310546875,
      "rewards/margins": 5.826567649841309,
      "rewards/rejected": -7.633131980895996,
      "step": 1504
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.428683625058341e-05,
      "logits/chosen": -1.7661545276641846,
      "logits/rejected": -1.7058085203170776,
      "logps/chosen": -182.27745056152344,
      "logps/rejected": -237.3101806640625,
      "loss": 0.069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6627051830291748,
      "rewards/margins": 7.234702110290527,
      "rewards/rejected": -8.897407531738281,
      "step": 1505
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.4254473625590942e-05,
      "logits/chosen": -1.9661635160446167,
      "logits/rejected": -1.9058352708816528,
      "logps/chosen": -155.5939178466797,
      "logps/rejected": -216.03033447265625,
      "loss": 0.0909,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7363879680633545,
      "rewards/margins": 7.684454917907715,
      "rewards/rejected": -8.420843124389648,
      "step": 1506
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.4222133072831143e-05,
      "logits/chosen": -1.903703212738037,
      "logits/rejected": -1.913364052772522,
      "logps/chosen": -197.48533630371094,
      "logps/rejected": -271.1314697265625,
      "loss": 0.0891,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.884815514087677,
      "rewards/margins": 7.804028511047363,
      "rewards/rejected": -8.688844680786133,
      "step": 1507
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.4189814658734302e-05,
      "logits/chosen": -2.017796754837036,
      "logits/rejected": -2.02762770652771,
      "logps/chosen": -179.27804565429688,
      "logps/rejected": -242.46725463867188,
      "loss": 0.0121,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1753333806991577,
      "rewards/margins": 6.9734907150268555,
      "rewards/rejected": -8.148824691772461,
      "step": 1508
    },
    {
      "epoch": 1.97,
      "learning_rate": 1.415751844968522e-05,
      "logits/chosen": -1.9961528778076172,
      "logits/rejected": -2.0355124473571777,
      "logps/chosen": -230.78713989257812,
      "logps/rejected": -287.765869140625,
      "loss": 0.1426,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7757691144943237,
      "rewards/margins": 5.198708534240723,
      "rewards/rejected": -6.974477291107178,
      "step": 1509
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.4125244512023062e-05,
      "logits/chosen": -1.7160990238189697,
      "logits/rejected": -1.6759029626846313,
      "logps/chosen": -201.62628173828125,
      "logps/rejected": -272.29205322265625,
      "loss": 0.0617,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4142707586288452,
      "rewards/margins": 7.462581634521484,
      "rewards/rejected": -8.876852035522461,
      "step": 1510
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.4092992912041274e-05,
      "logits/chosen": -1.855595588684082,
      "logits/rejected": -1.8531594276428223,
      "logps/chosen": -148.78341674804688,
      "logps/rejected": -206.61769104003906,
      "loss": 0.25,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2466044425964355,
      "rewards/margins": 5.727883338928223,
      "rewards/rejected": -6.974488258361816,
      "step": 1511
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.4060763715987418e-05,
      "logits/chosen": -1.8394930362701416,
      "logits/rejected": -1.8061363697052002,
      "logps/chosen": -162.24057006835938,
      "logps/rejected": -211.6754608154297,
      "loss": 0.1057,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7226440906524658,
      "rewards/margins": 6.253809452056885,
      "rewards/rejected": -6.97645378112793,
      "step": 1512
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.4028556990063018e-05,
      "logits/chosen": -1.9860260486602783,
      "logits/rejected": -1.9485723972320557,
      "logps/chosen": -204.5453338623047,
      "logps/rejected": -265.8670959472656,
      "loss": 0.1422,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.166626214981079,
      "rewards/margins": 6.418888568878174,
      "rewards/rejected": -7.585514545440674,
      "step": 1513
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.399637280042344e-05,
      "logits/chosen": -1.8057234287261963,
      "logits/rejected": -1.8638087511062622,
      "logps/chosen": -186.24363708496094,
      "logps/rejected": -260.63525390625,
      "loss": 0.0775,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8900512456893921,
      "rewards/margins": 7.869836330413818,
      "rewards/rejected": -8.7598876953125,
      "step": 1514
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.3964211213177777e-05,
      "logits/chosen": -1.5431137084960938,
      "logits/rejected": -1.528059720993042,
      "logps/chosen": -137.8041229248047,
      "logps/rejected": -237.1188201904297,
      "loss": 0.1076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7151025533676147,
      "rewards/margins": 7.016721725463867,
      "rewards/rejected": -8.731823921203613,
      "step": 1515
    },
    {
      "epoch": 1.98,
      "learning_rate": 1.3932072294388701e-05,
      "logits/chosen": -1.8800067901611328,
      "logits/rejected": -1.8036789894104004,
      "logps/chosen": -170.71678161621094,
      "logps/rejected": -224.3955078125,
      "loss": 0.0944,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3723101615905762,
      "rewards/margins": 5.514484882354736,
      "rewards/rejected": -6.8867950439453125,
      "step": 1516
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.3899956110072296e-05,
      "logits/chosen": -1.7704720497131348,
      "logits/rejected": -1.767439603805542,
      "logps/chosen": -142.60928344726562,
      "logps/rejected": -203.98214721679688,
      "loss": 0.1374,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8237273693084717,
      "rewards/margins": 5.416093349456787,
      "rewards/rejected": -7.23982048034668,
      "step": 1517
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.386786272619795e-05,
      "logits/chosen": -2.1056900024414062,
      "logits/rejected": -2.092741012573242,
      "logps/chosen": -157.19764709472656,
      "logps/rejected": -198.47091674804688,
      "loss": 0.1864,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9547339081764221,
      "rewards/margins": 4.672825813293457,
      "rewards/rejected": -5.627559185028076,
      "step": 1518
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.383579220868823e-05,
      "logits/chosen": -2.1192498207092285,
      "logits/rejected": -2.1118714809417725,
      "logps/chosen": -158.8693389892578,
      "logps/rejected": -212.41183471679688,
      "loss": 0.2254,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.8380639553070068,
      "rewards/margins": 5.388960838317871,
      "rewards/rejected": -7.227025032043457,
      "step": 1519
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.3803744623418751e-05,
      "logits/chosen": -1.8778517246246338,
      "logits/rejected": -1.8800277709960938,
      "logps/chosen": -193.85482788085938,
      "logps/rejected": -268.2300109863281,
      "loss": 0.0937,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8850143551826477,
      "rewards/margins": 7.058565139770508,
      "rewards/rejected": -7.943579196929932,
      "step": 1520
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.3771720036217969e-05,
      "logits/chosen": -1.9414777755737305,
      "logits/rejected": -1.9128940105438232,
      "logps/chosen": -168.39125061035156,
      "logps/rejected": -213.85501098632812,
      "loss": 0.0865,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.0075039863586426,
      "rewards/margins": 4.912435054779053,
      "rewards/rejected": -6.919939041137695,
      "step": 1521
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.3739718512867151e-05,
      "logits/chosen": -1.995743989944458,
      "logits/rejected": -2.0128209590911865,
      "logps/chosen": -217.19509887695312,
      "logps/rejected": -267.36688232421875,
      "loss": 0.1185,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2704944610595703,
      "rewards/margins": 6.018889904022217,
      "rewards/rejected": -7.289384841918945,
      "step": 1522
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.3707740119100185e-05,
      "logits/chosen": -1.949111819267273,
      "logits/rejected": -1.9382444620132446,
      "logps/chosen": -139.49795532226562,
      "logps/rejected": -185.3179931640625,
      "loss": 0.1946,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1124491691589355,
      "rewards/margins": 5.836550235748291,
      "rewards/rejected": -6.948999404907227,
      "step": 1523
    },
    {
      "epoch": 1.99,
      "learning_rate": 1.3675784920603397e-05,
      "logits/chosen": -1.8337342739105225,
      "logits/rejected": -1.8174057006835938,
      "logps/chosen": -180.5175018310547,
      "logps/rejected": -260.3408508300781,
      "loss": 0.1056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1399649381637573,
      "rewards/margins": 7.951875686645508,
      "rewards/rejected": -9.091839790344238,
      "step": 1524
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.3643852983015524e-05,
      "logits/chosen": -1.8588371276855469,
      "logits/rejected": -1.8975615501403809,
      "logps/chosen": -171.4891357421875,
      "logps/rejected": -227.4112091064453,
      "loss": 0.0388,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.163881778717041,
      "rewards/margins": 6.382929801940918,
      "rewards/rejected": -8.546812057495117,
      "step": 1525
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.3611944371927515e-05,
      "logits/chosen": -1.9417885541915894,
      "logits/rejected": -1.9384403228759766,
      "logps/chosen": -176.81021118164062,
      "logps/rejected": -237.068603515625,
      "loss": 0.057,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2735576331615448,
      "rewards/margins": 7.984800815582275,
      "rewards/rejected": -8.258358001708984,
      "step": 1526
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.3580059152882374e-05,
      "logits/chosen": -1.9223748445510864,
      "logits/rejected": -2.0598721504211426,
      "logps/chosen": -189.0762939453125,
      "logps/rejected": -300.93560791015625,
      "loss": 0.0457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9515939354896545,
      "rewards/margins": 9.141805648803711,
      "rewards/rejected": -10.093399047851562,
      "step": 1527
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.3548197391375092e-05,
      "logits/chosen": -1.790036916732788,
      "logits/rejected": -1.860375165939331,
      "logps/chosen": -168.46041870117188,
      "logps/rejected": -230.30291748046875,
      "loss": 0.1076,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5336273908615112,
      "rewards/margins": 6.87029504776001,
      "rewards/rejected": -8.403922080993652,
      "step": 1528
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.3516359152852443e-05,
      "logits/chosen": -1.775698184967041,
      "logits/rejected": -1.7525873184204102,
      "logps/chosen": -154.82923889160156,
      "logps/rejected": -237.45162963867188,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5750744342803955,
      "rewards/margins": 8.58332633972168,
      "rewards/rejected": -9.158400535583496,
      "step": 1529
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.348454450271292e-05,
      "logits/chosen": -1.597682237625122,
      "logits/rejected": -1.5742380619049072,
      "logps/chosen": -161.20761108398438,
      "logps/rejected": -241.42132568359375,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1535927653312683,
      "rewards/margins": 8.832878112792969,
      "rewards/rejected": -8.986471176147461,
      "step": 1530
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.345275350630652e-05,
      "logits/chosen": -2.0234615802764893,
      "logits/rejected": -1.991537094116211,
      "logps/chosen": -193.05874633789062,
      "logps/rejected": -302.61358642578125,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8610364198684692,
      "rewards/margins": 9.84919261932373,
      "rewards/rejected": -10.710229873657227,
      "step": 1531
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.342098622893469e-05,
      "logits/chosen": -1.8435871601104736,
      "logits/rejected": -1.8487372398376465,
      "logps/chosen": -180.15509033203125,
      "logps/rejected": -268.341796875,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7099978923797607,
      "rewards/margins": 9.448464393615723,
      "rewards/rejected": -10.158462524414062,
      "step": 1532
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.3389242735850146e-05,
      "logits/chosen": -1.820133924484253,
      "logits/rejected": -1.813864827156067,
      "logps/chosen": -234.839111328125,
      "logps/rejected": -329.2893371582031,
      "loss": 0.046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.37301284074783325,
      "rewards/margins": 8.691593170166016,
      "rewards/rejected": -9.064605712890625,
      "step": 1533
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.3357523092256742e-05,
      "logits/chosen": -1.8872143030166626,
      "logits/rejected": -1.8399633169174194,
      "logps/chosen": -192.03631591796875,
      "logps/rejected": -281.9556579589844,
      "loss": 0.005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.239000916481018,
      "rewards/margins": 8.872519493103027,
      "rewards/rejected": -10.111519813537598,
      "step": 1534
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.3325827363309329e-05,
      "logits/chosen": -1.8798092603683472,
      "logits/rejected": -1.8952518701553345,
      "logps/chosen": -144.6866455078125,
      "logps/rejected": -219.8625030517578,
      "loss": 0.046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.25402235984802246,
      "rewards/margins": 7.756320953369141,
      "rewards/rejected": -8.010343551635742,
      "step": 1535
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.3294155614113673e-05,
      "logits/chosen": -1.915588140487671,
      "logits/rejected": -1.8563159704208374,
      "logps/chosen": -205.7796630859375,
      "logps/rejected": -267.66900634765625,
      "loss": 0.0457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.40461376309394836,
      "rewards/margins": 7.57052755355835,
      "rewards/rejected": -7.9751410484313965,
      "step": 1536
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.3262507909726251e-05,
      "logits/chosen": -1.9503055810928345,
      "logits/rejected": -1.9627834558486938,
      "logps/chosen": -187.6611785888672,
      "logps/rejected": -248.67849731445312,
      "loss": 0.1316,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5386700630187988,
      "rewards/margins": 6.370505332946777,
      "rewards/rejected": -7.909175872802734,
      "step": 1537
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.3230884315154163e-05,
      "logits/chosen": -1.5061012506484985,
      "logits/rejected": -1.4742428064346313,
      "logps/chosen": -207.90957641601562,
      "logps/rejected": -254.99545288085938,
      "loss": 0.0876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2932624816894531,
      "rewards/margins": 6.949336051940918,
      "rewards/rejected": -8.242598533630371,
      "step": 1538
    },
    {
      "epoch": 2.01,
      "learning_rate": 1.3199284895355002e-05,
      "logits/chosen": -2.0590853691101074,
      "logits/rejected": -2.003415584564209,
      "logps/chosen": -161.7930450439453,
      "logps/rejected": -215.4407958984375,
      "loss": 0.0471,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6926572918891907,
      "rewards/margins": 7.323617458343506,
      "rewards/rejected": -8.016274452209473,
      "step": 1539
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.316770971523667e-05,
      "logits/chosen": -1.7058807611465454,
      "logits/rejected": -1.7154098749160767,
      "logps/chosen": -185.28135681152344,
      "logps/rejected": -271.00628662109375,
      "loss": 0.0019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9508638381958008,
      "rewards/margins": 8.433928489685059,
      "rewards/rejected": -9.384793281555176,
      "step": 1540
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.3136158839657287e-05,
      "logits/chosen": -1.7362006902694702,
      "logits/rejected": -1.8098485469818115,
      "logps/chosen": -165.66696166992188,
      "logps/rejected": -234.01658630371094,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9426742792129517,
      "rewards/margins": 7.938418865203857,
      "rewards/rejected": -8.88109302520752,
      "step": 1541
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.3104632333425066e-05,
      "logits/chosen": -1.8571577072143555,
      "logits/rejected": -1.8703685998916626,
      "logps/chosen": -153.8748016357422,
      "logps/rejected": -246.29823303222656,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6227169036865234,
      "rewards/margins": 8.549654960632324,
      "rewards/rejected": -9.172370910644531,
      "step": 1542
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.3073130261298167e-05,
      "logits/chosen": -2.0001440048217773,
      "logits/rejected": -2.03062105178833,
      "logps/chosen": -192.27334594726562,
      "logps/rejected": -261.22235107421875,
      "loss": 0.1325,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.7457628846168518,
      "rewards/margins": 6.937690734863281,
      "rewards/rejected": -7.68345308303833,
      "step": 1543
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.3041652687984535e-05,
      "logits/chosen": -2.0768544673919678,
      "logits/rejected": -2.042701244354248,
      "logps/chosen": -160.84426879882812,
      "logps/rejected": -249.868896484375,
      "loss": 0.0034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23251736164093018,
      "rewards/margins": 9.367364883422852,
      "rewards/rejected": -9.134847640991211,
      "step": 1544
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.3010199678141793e-05,
      "logits/chosen": -1.8806132078170776,
      "logits/rejected": -1.9293181896209717,
      "logps/chosen": -160.95156860351562,
      "logps/rejected": -239.3037109375,
      "loss": 0.0051,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6900899410247803,
      "rewards/margins": 8.367422103881836,
      "rewards/rejected": -9.057512283325195,
      "step": 1545
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.297877129637714e-05,
      "logits/chosen": -1.9400784969329834,
      "logits/rejected": -1.8593559265136719,
      "logps/chosen": -187.8778839111328,
      "logps/rejected": -259.97271728515625,
      "loss": 0.0043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.42858684062957764,
      "rewards/margins": 8.093515396118164,
      "rewards/rejected": -8.522103309631348,
      "step": 1546
    },
    {
      "epoch": 2.02,
      "learning_rate": 1.2947367607247168e-05,
      "logits/chosen": -1.797945261001587,
      "logits/rejected": -1.853203535079956,
      "logps/chosen": -149.82327270507812,
      "logps/rejected": -229.11961364746094,
      "loss": 0.0884,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5607883334159851,
      "rewards/margins": 7.72530460357666,
      "rewards/rejected": -8.286092758178711,
      "step": 1547
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.2915988675257729e-05,
      "logits/chosen": -1.9474494457244873,
      "logits/rejected": -1.9720731973648071,
      "logps/chosen": -173.7437744140625,
      "logps/rejected": -238.644287109375,
      "loss": 0.0482,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.059423804283142,
      "rewards/margins": 6.857656002044678,
      "rewards/rejected": -7.917079925537109,
      "step": 1548
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.2884634564863853e-05,
      "logits/chosen": -1.8357691764831543,
      "logits/rejected": -1.8078901767730713,
      "logps/chosen": -149.89376831054688,
      "logps/rejected": -234.67971801757812,
      "loss": 0.0912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.7086607813835144,
      "rewards/margins": 8.684245109558105,
      "rewards/rejected": -9.392906188964844,
      "step": 1549
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.2853305340469592e-05,
      "logits/chosen": -1.8318653106689453,
      "logits/rejected": -1.870279312133789,
      "logps/chosen": -177.15872192382812,
      "logps/rejected": -288.2128601074219,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.144337773323059,
      "rewards/margins": 9.237323760986328,
      "rewards/rejected": -10.381660461425781,
      "step": 1550
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.2822001066427818e-05,
      "logits/chosen": -1.9659843444824219,
      "logits/rejected": -1.857177972793579,
      "logps/chosen": -189.97537231445312,
      "logps/rejected": -259.89691162109375,
      "loss": 0.0889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.15678751468658447,
      "rewards/margins": 6.8027424812316895,
      "rewards/rejected": -6.959529399871826,
      "step": 1551
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.2790721807040216e-05,
      "logits/chosen": -1.4454537630081177,
      "logits/rejected": -1.4523166418075562,
      "logps/chosen": -190.044189453125,
      "logps/rejected": -288.3145751953125,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4206007719039917,
      "rewards/margins": 7.099836826324463,
      "rewards/rejected": -8.520437240600586,
      "step": 1552
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.2759467626557076e-05,
      "logits/chosen": -1.9478384256362915,
      "logits/rejected": -1.9578640460968018,
      "logps/chosen": -134.9126434326172,
      "logps/rejected": -198.01419067382812,
      "loss": 0.0974,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5989385843276978,
      "rewards/margins": 6.039154052734375,
      "rewards/rejected": -6.638092994689941,
      "step": 1553
    },
    {
      "epoch": 2.03,
      "learning_rate": 1.2728238589177141e-05,
      "logits/chosen": -1.8602335453033447,
      "logits/rejected": -1.8618937730789185,
      "logps/chosen": -162.0724639892578,
      "logps/rejected": -252.83335876464844,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6733867526054382,
      "rewards/margins": 9.5313720703125,
      "rewards/rejected": -10.20475959777832,
      "step": 1554
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.2697034759047561e-05,
      "logits/chosen": -2.031658887863159,
      "logits/rejected": -2.0899956226348877,
      "logps/chosen": -233.62115478515625,
      "logps/rejected": -317.1280517578125,
      "loss": 0.1307,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.4271396994590759,
      "rewards/margins": 7.834374904632568,
      "rewards/rejected": -8.261514663696289,
      "step": 1555
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.2665856200263649e-05,
      "logits/chosen": -1.9102786779403687,
      "logits/rejected": -1.9288216829299927,
      "logps/chosen": -159.32046508789062,
      "logps/rejected": -214.18898010253906,
      "loss": 0.0596,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8048596382141113,
      "rewards/margins": 6.3970866203308105,
      "rewards/rejected": -7.201946258544922,
      "step": 1556
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.2634702976868868e-05,
      "logits/chosen": -1.957202434539795,
      "logits/rejected": -2.0011603832244873,
      "logps/chosen": -168.7621307373047,
      "logps/rejected": -248.0120849609375,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6072189211845398,
      "rewards/margins": 7.88421106338501,
      "rewards/rejected": -8.491429328918457,
      "step": 1557
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.2603575152854582e-05,
      "logits/chosen": -1.9379693269729614,
      "logits/rejected": -2.017977237701416,
      "logps/chosen": -206.33309936523438,
      "logps/rejected": -296.46875,
      "loss": 0.0879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23542502522468567,
      "rewards/margins": 8.73104476928711,
      "rewards/rejected": -8.495619773864746,
      "step": 1558
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.2572472792160029e-05,
      "logits/chosen": -1.6278165578842163,
      "logits/rejected": -1.7174686193466187,
      "logps/chosen": -148.15188598632812,
      "logps/rejected": -217.92230224609375,
      "loss": 0.0464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9033560752868652,
      "rewards/margins": 6.628194808959961,
      "rewards/rejected": -7.531551361083984,
      "step": 1559
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.2541395958672128e-05,
      "logits/chosen": -1.8030297756195068,
      "logits/rejected": -1.7933604717254639,
      "logps/chosen": -190.2676239013672,
      "logps/rejected": -303.348388671875,
      "loss": 0.0453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9480107426643372,
      "rewards/margins": 7.992226600646973,
      "rewards/rejected": -8.940237998962402,
      "step": 1560
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.2510344716225353e-05,
      "logits/chosen": -2.020510196685791,
      "logits/rejected": -1.931838870048523,
      "logps/chosen": -157.22113037109375,
      "logps/rejected": -226.3065185546875,
      "loss": 0.046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9916752576828003,
      "rewards/margins": 7.9242024421691895,
      "rewards/rejected": -8.915878295898438,
      "step": 1561
    },
    {
      "epoch": 2.04,
      "learning_rate": 1.247931912860161e-05,
      "logits/chosen": -1.9241340160369873,
      "logits/rejected": -1.928293228149414,
      "logps/chosen": -148.0142822265625,
      "logps/rejected": -221.9215087890625,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3961063623428345,
      "rewards/margins": 6.953334808349609,
      "rewards/rejected": -7.349440574645996,
      "step": 1562
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.2448319259530129e-05,
      "logits/chosen": -2.0258774757385254,
      "logits/rejected": -1.9870299100875854,
      "logps/chosen": -170.74679565429688,
      "logps/rejected": -259.91046142578125,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.862501323223114,
      "rewards/margins": 8.945477485656738,
      "rewards/rejected": -9.807978630065918,
      "step": 1563
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.2417345172687303e-05,
      "logits/chosen": -1.766278862953186,
      "logits/rejected": -1.7836253643035889,
      "logps/chosen": -196.60377502441406,
      "logps/rejected": -267.83270263671875,
      "loss": 0.0927,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7093910574913025,
      "rewards/margins": 7.2185163497924805,
      "rewards/rejected": -7.927907943725586,
      "step": 1564
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.2386396931696545e-05,
      "logits/chosen": -1.8690192699432373,
      "logits/rejected": -1.8388170003890991,
      "logps/chosen": -245.1050262451172,
      "logps/rejected": -329.866455078125,
      "loss": 0.0463,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5733969211578369,
      "rewards/margins": 8.5335111618042,
      "rewards/rejected": -9.10690689086914,
      "step": 1565
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.235547460012822e-05,
      "logits/chosen": -1.780473232269287,
      "logits/rejected": -1.8153334856033325,
      "logps/chosen": -171.98928833007812,
      "logps/rejected": -257.4738464355469,
      "loss": 0.1305,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.22409774363040924,
      "rewards/margins": 7.66831636428833,
      "rewards/rejected": -7.892414093017578,
      "step": 1566
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.2324578241499434e-05,
      "logits/chosen": -1.9828410148620605,
      "logits/rejected": -1.9600255489349365,
      "logps/chosen": -158.23004150390625,
      "logps/rejected": -248.18252563476562,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22354663908481598,
      "rewards/margins": 9.188614845275879,
      "rewards/rejected": -9.412161827087402,
      "step": 1567
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.2293707919273951e-05,
      "logits/chosen": -2.0201985836029053,
      "logits/rejected": -2.0583760738372803,
      "logps/chosen": -140.92788696289062,
      "logps/rejected": -202.42965698242188,
      "loss": 0.0892,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1929229199886322,
      "rewards/margins": 6.620306015014648,
      "rewards/rejected": -6.813228607177734,
      "step": 1568
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.2262863696862067e-05,
      "logits/chosen": -1.5763084888458252,
      "logits/rejected": -1.5160216093063354,
      "logps/chosen": -229.78353881835938,
      "logps/rejected": -332.9212646484375,
      "loss": 0.0938,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.66045081615448,
      "rewards/margins": 8.19685173034668,
      "rewards/rejected": -8.857301712036133,
      "step": 1569
    },
    {
      "epoch": 2.05,
      "learning_rate": 1.223204563762047e-05,
      "logits/chosen": -1.7752225399017334,
      "logits/rejected": -1.899418592453003,
      "logps/chosen": -164.65655517578125,
      "logps/rejected": -228.12661743164062,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.320265293121338,
      "rewards/margins": 7.208536148071289,
      "rewards/rejected": -8.528801918029785,
      "step": 1570
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.2201253804852081e-05,
      "logits/chosen": -1.8807129859924316,
      "logits/rejected": -1.8088817596435547,
      "logps/chosen": -181.7335662841797,
      "logps/rejected": -265.099365234375,
      "loss": 0.0063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5922083854675293,
      "rewards/margins": 9.135930061340332,
      "rewards/rejected": -9.728137969970703,
      "step": 1571
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.2170488261805978e-05,
      "logits/chosen": -1.8305213451385498,
      "logits/rejected": -1.7762254476547241,
      "logps/chosen": -135.6221923828125,
      "logps/rejected": -210.20245361328125,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0456349849700928,
      "rewards/margins": 7.203673362731934,
      "rewards/rejected": -8.249308586120605,
      "step": 1572
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.2139749071677215e-05,
      "logits/chosen": -1.7544171810150146,
      "logits/rejected": -1.7707401514053345,
      "logps/chosen": -181.40780639648438,
      "logps/rejected": -285.5274353027344,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.37665632367134094,
      "rewards/margins": 9.727907180786133,
      "rewards/rejected": -10.10456371307373,
      "step": 1573
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.2109036297606733e-05,
      "logits/chosen": -1.8537029027938843,
      "logits/rejected": -1.902572751045227,
      "logps/chosen": -158.60977172851562,
      "logps/rejected": -228.5845184326172,
      "loss": 0.0077,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0519428625702858,
      "rewards/margins": 8.115686416625977,
      "rewards/rejected": -8.167628288269043,
      "step": 1574
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.207835000268119e-05,
      "logits/chosen": -1.387238621711731,
      "logits/rejected": -1.346356749534607,
      "logps/chosen": -151.66796875,
      "logps/rejected": -181.38385009765625,
      "loss": 0.1341,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.594295084476471,
      "rewards/margins": 5.817445278167725,
      "rewards/rejected": -6.411740303039551,
      "step": 1575
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.2047690249932881e-05,
      "logits/chosen": -1.731845498085022,
      "logits/rejected": -1.7612597942352295,
      "logps/chosen": -194.17115783691406,
      "logps/rejected": -272.23260498046875,
      "loss": 0.0504,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.268378496170044,
      "rewards/margins": 7.399502754211426,
      "rewards/rejected": -8.66788101196289,
      "step": 1576
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.2017057102339579e-05,
      "logits/chosen": -1.959835171699524,
      "logits/rejected": -1.9797602891921997,
      "logps/chosen": -191.00543212890625,
      "logps/rejected": -262.28778076171875,
      "loss": 0.0875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.097232699394226,
      "rewards/margins": 8.018265724182129,
      "rewards/rejected": -9.115497589111328,
      "step": 1577
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.198645062282436e-05,
      "logits/chosen": -1.8897427320480347,
      "logits/rejected": -1.9182778596878052,
      "logps/chosen": -144.8217315673828,
      "logps/rejected": -243.5197296142578,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6188120245933533,
      "rewards/margins": 8.994783401489258,
      "rewards/rejected": -9.613595962524414,
      "step": 1578
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.1955870874255581e-05,
      "logits/chosen": -1.7902731895446777,
      "logits/rejected": -1.739722728729248,
      "logps/chosen": -158.65203857421875,
      "logps/rejected": -257.3034973144531,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3603771924972534,
      "rewards/margins": 9.883246421813965,
      "rewards/rejected": -10.243623733520508,
      "step": 1579
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.1925317919446674e-05,
      "logits/chosen": -1.9108169078826904,
      "logits/rejected": -1.9132912158966064,
      "logps/chosen": -168.44456481933594,
      "logps/rejected": -267.6918029785156,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7670933604240417,
      "rewards/margins": 8.609723091125488,
      "rewards/rejected": -9.376815795898438,
      "step": 1580
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.189479182115601e-05,
      "logits/chosen": -1.553958773612976,
      "logits/rejected": -1.655795931816101,
      "logps/chosen": -155.0186767578125,
      "logps/rejected": -247.89962768554688,
      "loss": 0.0441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7563014626502991,
      "rewards/margins": 7.795615196228027,
      "rewards/rejected": -8.551916122436523,
      "step": 1581
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.1864292642086821e-05,
      "logits/chosen": -1.7375866174697876,
      "logits/rejected": -1.7598719596862793,
      "logps/chosen": -159.36813354492188,
      "logps/rejected": -230.33944702148438,
      "loss": 0.0466,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.706911027431488,
      "rewards/margins": 7.50990629196167,
      "rewards/rejected": -8.216817855834961,
      "step": 1582
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.1833820444887047e-05,
      "logits/chosen": -1.894447684288025,
      "logits/rejected": -1.8304792642593384,
      "logps/chosen": -167.99734497070312,
      "logps/rejected": -267.53082275390625,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2895434498786926,
      "rewards/margins": 10.26498031616211,
      "rewards/rejected": -9.975435256958008,
      "step": 1583
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.1803375292149188e-05,
      "logits/chosen": -2.0562291145324707,
      "logits/rejected": -2.054476499557495,
      "logps/chosen": -168.18017578125,
      "logps/rejected": -238.27259826660156,
      "loss": 0.0461,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6634958982467651,
      "rewards/margins": 8.41331672668457,
      "rewards/rejected": -9.076812744140625,
      "step": 1584
    },
    {
      "epoch": 2.07,
      "learning_rate": 1.1772957246410182e-05,
      "logits/chosen": -2.042128086090088,
      "logits/rejected": -1.9823732376098633,
      "logps/chosen": -142.55548095703125,
      "logps/rejected": -201.8201446533203,
      "loss": 0.185,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.3455074727535248,
      "rewards/margins": 6.1676411628723145,
      "rewards/rejected": -6.513149261474609,
      "step": 1585
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.174256637015132e-05,
      "logits/chosen": -1.9414622783660889,
      "logits/rejected": -1.962066888809204,
      "logps/chosen": -129.87710571289062,
      "logps/rejected": -207.21719360351562,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.26828640699386597,
      "rewards/margins": 7.6738128662109375,
      "rewards/rejected": -7.942099571228027,
      "step": 1586
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.1712202725798072e-05,
      "logits/chosen": -1.856528401374817,
      "logits/rejected": -1.8656179904937744,
      "logps/chosen": -203.07711791992188,
      "logps/rejected": -272.84844970703125,
      "loss": 0.0896,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6191532611846924,
      "rewards/margins": 7.336750030517578,
      "rewards/rejected": -8.955904006958008,
      "step": 1587
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.1681866375719962e-05,
      "logits/chosen": -1.9358348846435547,
      "logits/rejected": -2.0368244647979736,
      "logps/chosen": -186.8361358642578,
      "logps/rejected": -266.7251281738281,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.24728460609912872,
      "rewards/margins": 8.16058349609375,
      "rewards/rejected": -8.407867431640625,
      "step": 1588
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.1651557382230444e-05,
      "logits/chosen": -1.8691052198410034,
      "logits/rejected": -1.8194411993026733,
      "logps/chosen": -145.95245361328125,
      "logps/rejected": -207.54966735839844,
      "loss": 0.074,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0833581686019897,
      "rewards/margins": 6.91102409362793,
      "rewards/rejected": -7.994381904602051,
      "step": 1589
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.1621275807586799e-05,
      "logits/chosen": -1.8492367267608643,
      "logits/rejected": -1.748717188835144,
      "logps/chosen": -197.1922607421875,
      "logps/rejected": -293.1285400390625,
      "loss": 0.0877,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0229003429412842,
      "rewards/margins": 7.665094375610352,
      "rewards/rejected": -8.687994956970215,
      "step": 1590
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.1591021713989986e-05,
      "logits/chosen": -2.0967087745666504,
      "logits/rejected": -2.0782363414764404,
      "logps/chosen": -202.7138214111328,
      "logps/rejected": -279.8687744140625,
      "loss": 0.2167,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.5819624066352844,
      "rewards/margins": 8.620903015136719,
      "rewards/rejected": -9.202866554260254,
      "step": 1591
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.1560795163584492e-05,
      "logits/chosen": -1.6426539421081543,
      "logits/rejected": -1.7240560054779053,
      "logps/chosen": -169.92349243164062,
      "logps/rejected": -259.4862976074219,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5189768075942993,
      "rewards/margins": 8.072823524475098,
      "rewards/rejected": -9.591800689697266,
      "step": 1592
    },
    {
      "epoch": 2.08,
      "learning_rate": 1.153059621845825e-05,
      "logits/chosen": -1.8911799192428589,
      "logits/rejected": -1.8811068534851074,
      "logps/chosen": -163.4390869140625,
      "logps/rejected": -267.512939453125,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.637203574180603,
      "rewards/margins": 9.796133995056152,
      "rewards/rejected": -11.433338165283203,
      "step": 1593
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.1500424940642507e-05,
      "logits/chosen": -1.6580032110214233,
      "logits/rejected": -1.751070261001587,
      "logps/chosen": -142.73268127441406,
      "logps/rejected": -250.8894500732422,
      "loss": 0.0044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9399827122688293,
      "rewards/margins": 9.15585708618164,
      "rewards/rejected": -10.095840454101562,
      "step": 1594
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.1470281392111611e-05,
      "logits/chosen": -1.9321449995040894,
      "logits/rejected": -1.927699089050293,
      "logps/chosen": -163.82186889648438,
      "logps/rejected": -245.45162963867188,
      "loss": 0.0467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0471594333648682,
      "rewards/margins": 8.417715072631836,
      "rewards/rejected": -9.464874267578125,
      "step": 1595
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.144016563478302e-05,
      "logits/chosen": -2.0146875381469727,
      "logits/rejected": -1.9756101369857788,
      "logps/chosen": -172.3861541748047,
      "logps/rejected": -254.57325744628906,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.258683681488037,
      "rewards/margins": 7.366854667663574,
      "rewards/rejected": -8.62553882598877,
      "step": 1596
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.1410077730517089e-05,
      "logits/chosen": -1.9638988971710205,
      "logits/rejected": -1.9614040851593018,
      "logps/chosen": -154.13232421875,
      "logps/rejected": -262.3284606933594,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6503639221191406,
      "rewards/margins": 9.035061836242676,
      "rewards/rejected": -10.6854248046875,
      "step": 1597
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.1380017741116933e-05,
      "logits/chosen": -1.7888606786727905,
      "logits/rejected": -1.7171481847763062,
      "logps/chosen": -160.04055786132812,
      "logps/rejected": -235.05172729492188,
      "loss": 0.0456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1856850385665894,
      "rewards/margins": 7.730575084686279,
      "rewards/rejected": -8.916259765625,
      "step": 1598
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.134998572832837e-05,
      "logits/chosen": -1.764467477798462,
      "logits/rejected": -1.7621760368347168,
      "logps/chosen": -191.1162872314453,
      "logps/rejected": -306.0742492675781,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.39356309175491333,
      "rewards/margins": 9.410689353942871,
      "rewards/rejected": -9.804252624511719,
      "step": 1599
    },
    {
      "epoch": 2.09,
      "learning_rate": 1.1319981753839709e-05,
      "logits/chosen": -1.8391938209533691,
      "logits/rejected": -1.9397414922714233,
      "logps/chosen": -153.3414306640625,
      "logps/rejected": -245.9051513671875,
      "loss": 0.0486,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8426926136016846,
      "rewards/margins": 7.552426338195801,
      "rewards/rejected": -9.395118713378906,
      "step": 1600
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.129000587928171e-05,
      "logits/chosen": -1.8047947883605957,
      "logits/rejected": -1.7912423610687256,
      "logps/chosen": -184.89627075195312,
      "logps/rejected": -255.75331115722656,
      "loss": 0.1321,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0247026681900024,
      "rewards/margins": 6.320858478546143,
      "rewards/rejected": -7.3455610275268555,
      "step": 1601
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.1260058166227364e-05,
      "logits/chosen": -2.1134822368621826,
      "logits/rejected": -2.0303869247436523,
      "logps/chosen": -176.4046630859375,
      "logps/rejected": -217.1664276123047,
      "loss": 0.0444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9582228064537048,
      "rewards/margins": 7.328765392303467,
      "rewards/rejected": -8.286989212036133,
      "step": 1602
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.1230138676191857e-05,
      "logits/chosen": -1.6878161430358887,
      "logits/rejected": -1.7262670993804932,
      "logps/chosen": -158.12301635742188,
      "logps/rejected": -240.40267944335938,
      "loss": 0.0459,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.697560429573059,
      "rewards/margins": 7.02262544631958,
      "rewards/rejected": -8.720186233520508,
      "step": 1603
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.1200247470632393e-05,
      "logits/chosen": -1.9022256135940552,
      "logits/rejected": -1.8949040174484253,
      "logps/chosen": -218.2422637939453,
      "logps/rejected": -327.5567321777344,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4859800934791565,
      "rewards/margins": 9.000955581665039,
      "rewards/rejected": -9.48693561553955,
      "step": 1604
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.1170384610948065e-05,
      "logits/chosen": -1.9778677225112915,
      "logits/rejected": -2.024746894836426,
      "logps/chosen": -173.60293579101562,
      "logps/rejected": -255.98638916015625,
      "loss": 0.0468,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.31790199875831604,
      "rewards/margins": 7.503306865692139,
      "rewards/rejected": -7.821208953857422,
      "step": 1605
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.1140550158479737e-05,
      "logits/chosen": -1.938191533088684,
      "logits/rejected": -1.9275975227355957,
      "logps/chosen": -157.41958618164062,
      "logps/rejected": -258.36236572265625,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.016854166984558,
      "rewards/margins": 8.81126880645752,
      "rewards/rejected": -9.828123092651367,
      "step": 1606
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.1110744174509952e-05,
      "logits/chosen": -1.5897769927978516,
      "logits/rejected": -1.612013578414917,
      "logps/chosen": -181.90859985351562,
      "logps/rejected": -284.788818359375,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48295068740844727,
      "rewards/margins": 9.703218460083008,
      "rewards/rejected": -10.186169624328613,
      "step": 1607
    },
    {
      "epoch": 2.1,
      "learning_rate": 1.1080966720262737e-05,
      "logits/chosen": -1.9939075708389282,
      "logits/rejected": -1.9334222078323364,
      "logps/chosen": -191.09689331054688,
      "logps/rejected": -244.90097045898438,
      "loss": 0.0476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.131523609161377,
      "rewards/margins": 7.956849098205566,
      "rewards/rejected": -9.088373184204102,
      "step": 1608
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.1051217856903551e-05,
      "logits/chosen": -1.9125679731369019,
      "logits/rejected": -1.981872320175171,
      "logps/chosen": -141.1490478515625,
      "logps/rejected": -219.94744873046875,
      "loss": 0.0881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.216914415359497,
      "rewards/margins": 6.86641788482666,
      "rewards/rejected": -8.083332061767578,
      "step": 1609
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.1021497645539115e-05,
      "logits/chosen": -1.9560807943344116,
      "logits/rejected": -1.9967155456542969,
      "logps/chosen": -145.06689453125,
      "logps/rejected": -229.4589385986328,
      "loss": 0.0447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.914740800857544,
      "rewards/margins": 7.498084545135498,
      "rewards/rejected": -8.412824630737305,
      "step": 1610
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.0991806147217282e-05,
      "logits/chosen": -1.8740688562393188,
      "logits/rejected": -1.8333930969238281,
      "logps/chosen": -167.72760009765625,
      "logps/rejected": -257.2799072265625,
      "loss": 0.0454,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4589805006980896,
      "rewards/margins": 9.744257926940918,
      "rewards/rejected": -10.203237533569336,
      "step": 1611
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.0962143422926929e-05,
      "logits/chosen": -1.5847423076629639,
      "logits/rejected": -1.6017591953277588,
      "logps/chosen": -190.1009521484375,
      "logps/rejected": -265.97235107421875,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4791683554649353,
      "rewards/margins": 9.916194915771484,
      "rewards/rejected": -10.395363807678223,
      "step": 1612
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.0932509533597843e-05,
      "logits/chosen": -1.7772189378738403,
      "logits/rejected": -1.8497400283813477,
      "logps/chosen": -157.90924072265625,
      "logps/rejected": -266.33984375,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4629042446613312,
      "rewards/margins": 9.880802154541016,
      "rewards/rejected": -10.343706130981445,
      "step": 1613
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.0902904540100587e-05,
      "logits/chosen": -2.0442440509796143,
      "logits/rejected": -2.050283908843994,
      "logps/chosen": -155.6942138671875,
      "logps/rejected": -229.55810546875,
      "loss": 0.1304,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.44657114148139954,
      "rewards/margins": 7.513418674468994,
      "rewards/rejected": -7.95999002456665,
      "step": 1614
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.0873328503246336e-05,
      "logits/chosen": -1.8808633089065552,
      "logits/rejected": -1.8794891834259033,
      "logps/chosen": -153.5122528076172,
      "logps/rejected": -242.74119567871094,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3033973276615143,
      "rewards/margins": 9.738836288452148,
      "rewards/rejected": -9.435440063476562,
      "step": 1615
    },
    {
      "epoch": 2.11,
      "learning_rate": 1.0843781483786823e-05,
      "logits/chosen": -1.6431313753128052,
      "logits/rejected": -1.6379454135894775,
      "logps/chosen": -164.75640869140625,
      "logps/rejected": -240.2802276611328,
      "loss": 0.0469,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9619613885879517,
      "rewards/margins": 8.21091079711914,
      "rewards/rejected": -9.172873497009277,
      "step": 1616
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.081426354241414e-05,
      "logits/chosen": -1.7526963949203491,
      "logits/rejected": -1.8265552520751953,
      "logps/chosen": -201.28782653808594,
      "logps/rejected": -278.186279296875,
      "loss": 0.1066,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.644562005996704,
      "rewards/margins": 7.069887161254883,
      "rewards/rejected": -8.714448928833008,
      "step": 1617
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.0784774739760694e-05,
      "logits/chosen": -1.7705386877059937,
      "logits/rejected": -1.7747803926467896,
      "logps/chosen": -174.0726318359375,
      "logps/rejected": -201.94419860839844,
      "loss": 0.0915,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8033338785171509,
      "rewards/margins": 6.533467769622803,
      "rewards/rejected": -7.336801528930664,
      "step": 1618
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.075531513639899e-05,
      "logits/chosen": -1.8598980903625488,
      "logits/rejected": -2.0005438327789307,
      "logps/chosen": -166.49180603027344,
      "logps/rejected": -293.18841552734375,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9469296336174011,
      "rewards/margins": 10.362959861755371,
      "rewards/rejected": -11.30988883972168,
      "step": 1619
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.0725884792841598e-05,
      "logits/chosen": -1.8182711601257324,
      "logits/rejected": -1.8281532526016235,
      "logps/chosen": -175.6002197265625,
      "logps/rejected": -259.384033203125,
      "loss": 0.087,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6488322615623474,
      "rewards/margins": 9.423254013061523,
      "rewards/rejected": -10.072086334228516,
      "step": 1620
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.0696483769540974e-05,
      "logits/chosen": -1.593421220779419,
      "logits/rejected": -1.6254453659057617,
      "logps/chosen": -158.62582397460938,
      "logps/rejected": -260.86920166015625,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.392691969871521,
      "rewards/margins": 9.465681076049805,
      "rewards/rejected": -10.858373641967773,
      "step": 1621
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.0667112126889314e-05,
      "logits/chosen": -1.9771430492401123,
      "logits/rejected": -2.0816547870635986,
      "logps/chosen": -166.35061645507812,
      "logps/rejected": -260.0909423828125,
      "loss": 0.1997,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.483760952949524,
      "rewards/margins": 7.217160701751709,
      "rewards/rejected": -8.700921058654785,
      "step": 1622
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.0637769925218502e-05,
      "logits/chosen": -1.8516621589660645,
      "logits/rejected": -1.8745663166046143,
      "logps/chosen": -161.97296142578125,
      "logps/rejected": -238.10736083984375,
      "loss": 0.0511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1518847942352295,
      "rewards/margins": 7.468740463256836,
      "rewards/rejected": -9.620625495910645,
      "step": 1623
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0608457224799953e-05,
      "logits/chosen": -1.7654082775115967,
      "logits/rejected": -1.7879576683044434,
      "logps/chosen": -161.9789581298828,
      "logps/rejected": -238.18296813964844,
      "loss": 0.0553,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1331989765167236,
      "rewards/margins": 7.13073205947876,
      "rewards/rejected": -9.263930320739746,
      "step": 1624
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0579174085844442e-05,
      "logits/chosen": -2.0330822467803955,
      "logits/rejected": -2.023556709289551,
      "logps/chosen": -176.66212463378906,
      "logps/rejected": -280.760009765625,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.15543267130851746,
      "rewards/margins": 9.975716590881348,
      "rewards/rejected": -10.131149291992188,
      "step": 1625
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0549920568502065e-05,
      "logits/chosen": -1.7348453998565674,
      "logits/rejected": -1.7109066247940063,
      "logps/chosen": -158.89273071289062,
      "logps/rejected": -225.9927215576172,
      "loss": 0.092,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7510828971862793,
      "rewards/margins": 6.84454345703125,
      "rewards/rejected": -8.595625877380371,
      "step": 1626
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0520696732862057e-05,
      "logits/chosen": -1.9483460187911987,
      "logits/rejected": -1.9873994588851929,
      "logps/chosen": -176.2453155517578,
      "logps/rejected": -243.611328125,
      "loss": 0.1353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.401078701019287,
      "rewards/margins": 7.193070888519287,
      "rewards/rejected": -8.594149589538574,
      "step": 1627
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0491502638952675e-05,
      "logits/chosen": -1.745213270187378,
      "logits/rejected": -1.7260756492614746,
      "logps/chosen": -172.40028381347656,
      "logps/rejected": -233.21083068847656,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04702691733837128,
      "rewards/margins": 8.451311111450195,
      "rewards/rejected": -8.404284477233887,
      "step": 1628
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0462338346741086e-05,
      "logits/chosen": -1.8193237781524658,
      "logits/rejected": -1.8550066947937012,
      "logps/chosen": -162.42617797851562,
      "logps/rejected": -260.2054748535156,
      "loss": 0.044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.630279302597046,
      "rewards/margins": 8.321517944335938,
      "rewards/rejected": -9.951796531677246,
      "step": 1629
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0433203916133252e-05,
      "logits/chosen": -1.8887048959732056,
      "logits/rejected": -1.9195616245269775,
      "logps/chosen": -157.6302490234375,
      "logps/rejected": -235.54953002929688,
      "loss": 0.0451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.854744553565979,
      "rewards/margins": 8.505916595458984,
      "rewards/rejected": -9.360661506652832,
      "step": 1630
    },
    {
      "epoch": 2.13,
      "learning_rate": 1.0404099406973803e-05,
      "logits/chosen": -2.018538475036621,
      "logits/rejected": -2.068702459335327,
      "logps/chosen": -172.5665283203125,
      "logps/rejected": -265.7096252441406,
      "loss": 0.0106,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0310336351394653,
      "rewards/margins": 8.395581245422363,
      "rewards/rejected": -9.426614761352539,
      "step": 1631
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.0375024879045889e-05,
      "logits/chosen": -2.0561649799346924,
      "logits/rejected": -1.9543439149856567,
      "logps/chosen": -249.31475830078125,
      "logps/rejected": -317.0854797363281,
      "loss": 0.0436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7450178861618042,
      "rewards/margins": 9.634797096252441,
      "rewards/rejected": -10.379816055297852,
      "step": 1632
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.0345980392071073e-05,
      "logits/chosen": -1.7793387174606323,
      "logits/rejected": -1.7165791988372803,
      "logps/chosen": -154.4380340576172,
      "logps/rejected": -220.67208862304688,
      "loss": 0.0877,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5957815647125244,
      "rewards/margins": 7.864494800567627,
      "rewards/rejected": -8.46027660369873,
      "step": 1633
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.031696600570923e-05,
      "logits/chosen": -1.856867790222168,
      "logits/rejected": -1.8572534322738647,
      "logps/chosen": -135.80673217773438,
      "logps/rejected": -229.3101348876953,
      "loss": 0.1303,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.8688921332359314,
      "rewards/margins": 8.788094520568848,
      "rewards/rejected": -9.656986236572266,
      "step": 1634
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.0287981779558411e-05,
      "logits/chosen": -2.0594406127929688,
      "logits/rejected": -2.072843551635742,
      "logps/chosen": -222.73760986328125,
      "logps/rejected": -263.98828125,
      "loss": 0.1406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4687888622283936,
      "rewards/margins": 5.454514026641846,
      "rewards/rejected": -6.923303127288818,
      "step": 1635
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.0259027773154681e-05,
      "logits/chosen": -2.025172710418701,
      "logits/rejected": -2.1201062202453613,
      "logps/chosen": -155.17721557617188,
      "logps/rejected": -236.1791229248047,
      "loss": 0.0903,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1302742063999176,
      "rewards/margins": 8.581464767456055,
      "rewards/rejected": -8.451190948486328,
      "step": 1636
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.023010404597206e-05,
      "logits/chosen": -1.8177903890609741,
      "logits/rejected": -1.8465934991836548,
      "logps/chosen": -152.30001831054688,
      "logps/rejected": -252.5247802734375,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3091137409210205,
      "rewards/margins": 9.514116287231445,
      "rewards/rejected": -10.823230743408203,
      "step": 1637
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.0201210657422386e-05,
      "logits/chosen": -1.561299443244934,
      "logits/rejected": -1.6054558753967285,
      "logps/chosen": -178.1842041015625,
      "logps/rejected": -304.1603088378906,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.3985576033592224,
      "rewards/margins": 10.361804962158203,
      "rewards/rejected": -10.760363578796387,
      "step": 1638
    },
    {
      "epoch": 2.14,
      "learning_rate": 1.0172347666855117e-05,
      "logits/chosen": -1.7882592678070068,
      "logits/rejected": -1.7832262516021729,
      "logps/chosen": -146.73507690429688,
      "logps/rejected": -238.3235626220703,
      "loss": 0.0583,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6074613332748413,
      "rewards/margins": 8.448479652404785,
      "rewards/rejected": -9.055940628051758,
      "step": 1639
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.0143515133557333e-05,
      "logits/chosen": -2.0044748783111572,
      "logits/rejected": -2.04290771484375,
      "logps/chosen": -166.8870086669922,
      "logps/rejected": -246.28811645507812,
      "loss": 0.088,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.6391198635101318,
      "rewards/margins": 7.364441871643066,
      "rewards/rejected": -9.003561019897461,
      "step": 1640
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.0114713116753533e-05,
      "logits/chosen": -1.8734256029129028,
      "logits/rejected": -1.9572198390960693,
      "logps/chosen": -149.54124450683594,
      "logps/rejected": -245.07351684570312,
      "loss": 0.1304,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5698168277740479,
      "rewards/margins": 8.120656967163086,
      "rewards/rejected": -8.690473556518555,
      "step": 1641
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.0085941675605517e-05,
      "logits/chosen": -1.9102582931518555,
      "logits/rejected": -1.957924485206604,
      "logps/chosen": -155.18978881835938,
      "logps/rejected": -257.0257568359375,
      "loss": 0.0885,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03462205082178116,
      "rewards/margins": 8.777915954589844,
      "rewards/rejected": -8.743293762207031,
      "step": 1642
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.0057200869212308e-05,
      "logits/chosen": -1.7796900272369385,
      "logits/rejected": -1.7574498653411865,
      "logps/chosen": -177.71652221679688,
      "logps/rejected": -248.51292419433594,
      "loss": 0.0923,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1380293369293213,
      "rewards/margins": 7.498992443084717,
      "rewards/rejected": -8.637022018432617,
      "step": 1643
    },
    {
      "epoch": 2.15,
      "learning_rate": 1.0028490756609971e-05,
      "logits/chosen": -1.9873998165130615,
      "logits/rejected": -2.005246162414551,
      "logps/chosen": -165.5908660888672,
      "logps/rejected": -264.8787536621094,
      "loss": 0.0462,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2477751970291138,
      "rewards/margins": 7.637832164764404,
      "rewards/rejected": -8.885608673095703,
      "step": 1644
    },
    {
      "epoch": 2.15,
      "learning_rate": 9.999811396771554e-06,
      "logits/chosen": -1.9696788787841797,
      "logits/rejected": -2.0444154739379883,
      "logps/chosen": -126.6568832397461,
      "logps/rejected": -210.0042724609375,
      "loss": 0.089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9937084913253784,
      "rewards/margins": 7.025668621063232,
      "rewards/rejected": -8.019376754760742,
      "step": 1645
    },
    {
      "epoch": 2.15,
      "learning_rate": 9.971162848606907e-06,
      "logits/chosen": -1.625485897064209,
      "logits/rejected": -1.5573269128799438,
      "logps/chosen": -152.0515594482422,
      "logps/rejected": -238.1817626953125,
      "loss": 0.0048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.009362727403640747,
      "rewards/margins": 9.353007316589355,
      "rewards/rejected": -9.362370491027832,
      "step": 1646
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.942545170962611e-06,
      "logits/chosen": -2.057633876800537,
      "logits/rejected": -2.09076189994812,
      "logps/chosen": -174.6454620361328,
      "logps/rejected": -252.92984008789062,
      "loss": 0.0876,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4979091882705688,
      "rewards/margins": 7.134169578552246,
      "rewards/rejected": -8.632078170776367,
      "step": 1647
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.913958422621845e-06,
      "logits/chosen": -1.8510873317718506,
      "logits/rejected": -1.8383934497833252,
      "logps/chosen": -234.2716064453125,
      "logps/rejected": -292.3934326171875,
      "loss": 0.0456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1968002319335938,
      "rewards/margins": 8.13193130493164,
      "rewards/rejected": -10.328730583190918,
      "step": 1648
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.885402662304222e-06,
      "logits/chosen": -2.034956693649292,
      "logits/rejected": -2.055487632751465,
      "logps/chosen": -170.5280303955078,
      "logps/rejected": -294.7333068847656,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1446633338928223,
      "rewards/margins": 9.01904010772705,
      "rewards/rejected": -10.163703918457031,
      "step": 1649
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.856877948665724e-06,
      "logits/chosen": -1.6261368989944458,
      "logits/rejected": -1.5786309242248535,
      "logps/chosen": -220.7260284423828,
      "logps/rejected": -266.1442565917969,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.48670896887779236,
      "rewards/margins": 9.230504989624023,
      "rewards/rejected": -9.717214584350586,
      "step": 1650
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.828384340298572e-06,
      "logits/chosen": -1.9709876775741577,
      "logits/rejected": -1.9152164459228516,
      "logps/chosen": -156.30288696289062,
      "logps/rejected": -236.9882354736328,
      "loss": 0.0874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0939602851867676,
      "rewards/margins": 8.413116455078125,
      "rewards/rejected": -9.507076263427734,
      "step": 1651
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.799921895731062e-06,
      "logits/chosen": -1.7274540662765503,
      "logits/rejected": -1.7755210399627686,
      "logps/chosen": -122.5679931640625,
      "logps/rejected": -214.5566864013672,
      "loss": 0.0462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5203177332878113,
      "rewards/margins": 8.465195655822754,
      "rewards/rejected": -8.985512733459473,
      "step": 1652
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.771490673427508e-06,
      "logits/chosen": -1.8614534139633179,
      "logits/rejected": -1.9787870645523071,
      "logps/chosen": -170.04945373535156,
      "logps/rejected": -277.22686767578125,
      "loss": 0.0441,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0692503452301025,
      "rewards/margins": 9.166557312011719,
      "rewards/rejected": -10.235808372497559,
      "step": 1653
    },
    {
      "epoch": 2.16,
      "learning_rate": 9.743090731788088e-06,
      "logits/chosen": -1.680272102355957,
      "logits/rejected": -1.7287495136260986,
      "logps/chosen": -158.24078369140625,
      "logps/rejected": -245.72885131835938,
      "loss": 0.0451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1765902042388916,
      "rewards/margins": 8.263666152954102,
      "rewards/rejected": -9.440256118774414,
      "step": 1654
    },
    {
      "epoch": 2.17,
      "learning_rate": 9.714722129148705e-06,
      "logits/chosen": -1.7114430665969849,
      "logits/rejected": -1.7526739835739136,
      "logps/chosen": -180.62680053710938,
      "logps/rejected": -224.20797729492188,
      "loss": 0.0883,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6282711625099182,
      "rewards/margins": 7.708086013793945,
      "rewards/rejected": -8.336357116699219,
      "step": 1655
    },
    {
      "epoch": 2.17,
      "learning_rate": 9.686384923780894e-06,
      "logits/chosen": -1.8871158361434937,
      "logits/rejected": -1.8732534646987915,
      "logps/chosen": -175.20289611816406,
      "logps/rejected": -243.34829711914062,
      "loss": 0.0453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7468876242637634,
      "rewards/margins": 7.832474708557129,
      "rewards/rejected": -8.579360961914062,
      "step": 1656
    },
    {
      "epoch": 2.17,
      "learning_rate": 9.658079173891718e-06,
      "logits/chosen": -1.6555521488189697,
      "logits/rejected": -1.6963847875595093,
      "logps/chosen": -217.43209838867188,
      "logps/rejected": -310.2213134765625,
      "loss": 0.0457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.887463092803955,
      "rewards/margins": 8.336713790893555,
      "rewards/rejected": -10.224178314208984,
      "step": 1657
    },
    {
      "epoch": 2.17,
      "learning_rate": 9.62980493762362e-06,
      "logits/chosen": -1.970055103302002,
      "logits/rejected": -1.984239101409912,
      "logps/chosen": -149.95306396484375,
      "logps/rejected": -246.7270050048828,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3183818459510803,
      "rewards/margins": 9.318705558776855,
      "rewards/rejected": -9.63708782196045,
      "step": 1658
    },
    {
      "epoch": 2.17,
      "learning_rate": 9.60156227305429e-06,
      "logits/chosen": -1.8576709032058716,
      "logits/rejected": -1.834694743156433,
      "logps/chosen": -162.2405548095703,
      "logps/rejected": -229.83279418945312,
      "loss": 0.0072,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3267923593521118,
      "rewards/margins": 7.084452152252197,
      "rewards/rejected": -8.41124439239502,
      "step": 1659
    },
    {
      "epoch": 2.17,
      "learning_rate": 9.573351238196598e-06,
      "logits/chosen": -2.0115129947662354,
      "logits/rejected": -1.981108546257019,
      "logps/chosen": -169.37738037109375,
      "logps/rejected": -258.3280944824219,
      "loss": 0.089,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5738982558250427,
      "rewards/margins": 9.526470184326172,
      "rewards/rejected": -10.100367546081543,
      "step": 1660
    },
    {
      "epoch": 2.17,
      "learning_rate": 9.545171890998415e-06,
      "logits/chosen": -1.5641111135482788,
      "logits/rejected": -1.5682613849639893,
      "logps/chosen": -153.1749725341797,
      "logps/rejected": -232.18116760253906,
      "loss": 0.1322,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7605468034744263,
      "rewards/margins": 7.3993754386901855,
      "rewards/rejected": -9.15992259979248,
      "step": 1661
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.51702428934255e-06,
      "logits/chosen": -1.5826655626296997,
      "logits/rejected": -1.6123006343841553,
      "logps/chosen": -149.99392700195312,
      "logps/rejected": -239.59884643554688,
      "loss": 0.0026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0853947401046753,
      "rewards/margins": 8.840001106262207,
      "rewards/rejected": -9.925395965576172,
      "step": 1662
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.488908491046575e-06,
      "logits/chosen": -1.5909959077835083,
      "logits/rejected": -1.672724723815918,
      "logps/chosen": -171.6544952392578,
      "logps/rejected": -268.6630859375,
      "loss": 0.0047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2057874202728271,
      "rewards/margins": 9.563070297241211,
      "rewards/rejected": -10.768857955932617,
      "step": 1663
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.460824553862762e-06,
      "logits/chosen": -1.9411277770996094,
      "logits/rejected": -1.940759539604187,
      "logps/chosen": -157.8964080810547,
      "logps/rejected": -251.29319763183594,
      "loss": 0.046,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4408135414123535,
      "rewards/margins": 8.439933776855469,
      "rewards/rejected": -8.880746841430664,
      "step": 1664
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.432772535477941e-06,
      "logits/chosen": -1.9864856004714966,
      "logits/rejected": -2.0316200256347656,
      "logps/chosen": -146.45347595214844,
      "logps/rejected": -260.033935546875,
      "loss": 0.0262,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5745049715042114,
      "rewards/margins": 9.994059562683105,
      "rewards/rejected": -10.568563461303711,
      "step": 1665
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.40475249351333e-06,
      "logits/chosen": -2.049938678741455,
      "logits/rejected": -2.0287792682647705,
      "logps/chosen": -151.88558959960938,
      "logps/rejected": -219.84323120117188,
      "loss": 0.131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6135196685791016,
      "rewards/margins": 7.2162885665893555,
      "rewards/rejected": -7.829808235168457,
      "step": 1666
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.376764485524515e-06,
      "logits/chosen": -1.8999272584915161,
      "logits/rejected": -1.929114818572998,
      "logps/chosen": -194.48121643066406,
      "logps/rejected": -257.4184875488281,
      "loss": 0.1308,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.7833256721496582,
      "rewards/margins": 6.35922908782959,
      "rewards/rejected": -8.142553329467773,
      "step": 1667
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.348808569001272e-06,
      "logits/chosen": -1.7413606643676758,
      "logits/rejected": -1.7168065309524536,
      "logps/chosen": -182.652099609375,
      "logps/rejected": -266.08135986328125,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.448183298110962,
      "rewards/margins": 9.240997314453125,
      "rewards/rejected": -10.689179420471191,
      "step": 1668
    },
    {
      "epoch": 2.18,
      "learning_rate": 9.320884801367435e-06,
      "logits/chosen": -1.7820024490356445,
      "logits/rejected": -1.737465500831604,
      "logps/chosen": -169.5586700439453,
      "logps/rejected": -262.9524841308594,
      "loss": 0.0069,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2813034057617188,
      "rewards/margins": 8.11032772064209,
      "rewards/rejected": -9.391630172729492,
      "step": 1669
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.292993239980827e-06,
      "logits/chosen": -1.7238365411758423,
      "logits/rejected": -1.7120840549468994,
      "logps/chosen": -170.41986083984375,
      "logps/rejected": -258.7507629394531,
      "loss": 0.0915,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9819586277008057,
      "rewards/margins": 7.511738300323486,
      "rewards/rejected": -9.493697166442871,
      "step": 1670
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.265133942133115e-06,
      "logits/chosen": -2.0185389518737793,
      "logits/rejected": -2.023789167404175,
      "logps/chosen": -164.50442504882812,
      "logps/rejected": -282.65325927734375,
      "loss": 0.0074,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6014794111251831,
      "rewards/margins": 10.601015090942383,
      "rewards/rejected": -11.202495574951172,
      "step": 1671
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.237306965049677e-06,
      "logits/chosen": -1.3205431699752808,
      "logits/rejected": -1.2434921264648438,
      "logps/chosen": -163.4990234375,
      "logps/rejected": -270.09539794921875,
      "loss": 0.1334,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6097871661186218,
      "rewards/margins": 10.299297332763672,
      "rewards/rejected": -10.909085273742676,
      "step": 1672
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.2095123658895e-06,
      "logits/chosen": -1.9109156131744385,
      "logits/rejected": -1.8916103839874268,
      "logps/chosen": -270.3733215332031,
      "logps/rejected": -336.1594543457031,
      "loss": 0.1307,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9747849106788635,
      "rewards/margins": 7.436781883239746,
      "rewards/rejected": -8.411566734313965,
      "step": 1673
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.181750201745087e-06,
      "logits/chosen": -1.679357647895813,
      "logits/rejected": -1.6985632181167603,
      "logps/chosen": -181.4761199951172,
      "logps/rejected": -216.17796325683594,
      "loss": 0.1342,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.9446721076965332,
      "rewards/margins": 5.449231147766113,
      "rewards/rejected": -7.3939032554626465,
      "step": 1674
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.15402052964231e-06,
      "logits/chosen": -1.7306880950927734,
      "logits/rejected": -1.727543830871582,
      "logps/chosen": -166.93174743652344,
      "logps/rejected": -254.4122772216797,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.070855975151062,
      "rewards/margins": 8.598644256591797,
      "rewards/rejected": -9.669500350952148,
      "step": 1675
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.126323406540282e-06,
      "logits/chosen": -2.052203416824341,
      "logits/rejected": -2.1164824962615967,
      "logps/chosen": -184.93821716308594,
      "logps/rejected": -271.6320495605469,
      "loss": 0.1306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5415613651275635,
      "rewards/margins": 8.265079498291016,
      "rewards/rejected": -8.806641578674316,
      "step": 1676
    },
    {
      "epoch": 2.19,
      "learning_rate": 9.098658889331265e-06,
      "logits/chosen": -1.9378398656845093,
      "logits/rejected": -1.9936093091964722,
      "logps/chosen": -203.70806884765625,
      "logps/rejected": -298.83282470703125,
      "loss": 0.0476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.304976463317871,
      "rewards/margins": 8.67045783996582,
      "rewards/rejected": -10.975434303283691,
      "step": 1677
    },
    {
      "epoch": 2.2,
      "learning_rate": 9.07102703484056e-06,
      "logits/chosen": -1.7480404376983643,
      "logits/rejected": -1.7458223104476929,
      "logps/chosen": -154.57672119140625,
      "logps/rejected": -249.1329345703125,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19518813490867615,
      "rewards/margins": 9.094565391540527,
      "rewards/rejected": -9.289752960205078,
      "step": 1678
    },
    {
      "epoch": 2.2,
      "learning_rate": 9.043427899826367e-06,
      "logits/chosen": -1.5898010730743408,
      "logits/rejected": -1.562047004699707,
      "logps/chosen": -209.6298065185547,
      "logps/rejected": -300.7589416503906,
      "loss": 0.0474,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0362350940704346,
      "rewards/margins": 7.6428961753845215,
      "rewards/rejected": -9.679130554199219,
      "step": 1679
    },
    {
      "epoch": 2.2,
      "learning_rate": 9.015861540979667e-06,
      "logits/chosen": -2.0676355361938477,
      "logits/rejected": -2.0460362434387207,
      "logps/chosen": -167.3324737548828,
      "logps/rejected": -251.31936645507812,
      "loss": 0.0471,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3568321466445923,
      "rewards/margins": 8.01896858215332,
      "rewards/rejected": -9.375800132751465,
      "step": 1680
    },
    {
      "epoch": 2.2,
      "learning_rate": 8.988328014924136e-06,
      "logits/chosen": -1.8934491872787476,
      "logits/rejected": -1.8987008333206177,
      "logps/chosen": -173.1253662109375,
      "logps/rejected": -232.80796813964844,
      "loss": 0.0895,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.673693299293518,
      "rewards/margins": 7.304157733917236,
      "rewards/rejected": -8.977851867675781,
      "step": 1681
    },
    {
      "epoch": 2.2,
      "learning_rate": 8.960827378215994e-06,
      "logits/chosen": -1.6453583240509033,
      "logits/rejected": -1.621715784072876,
      "logps/chosen": -160.49807739257812,
      "logps/rejected": -254.8524627685547,
      "loss": 0.0026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0426502227783203,
      "rewards/margins": 8.370094299316406,
      "rewards/rejected": -10.412744522094727,
      "step": 1682
    },
    {
      "epoch": 2.2,
      "learning_rate": 8.933359687343895e-06,
      "logits/chosen": -1.9197800159454346,
      "logits/rejected": -1.9210864305496216,
      "logps/chosen": -184.84051513671875,
      "logps/rejected": -272.6552734375,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.28548863530158997,
      "rewards/margins": 10.919346809387207,
      "rewards/rejected": -11.204835891723633,
      "step": 1683
    },
    {
      "epoch": 2.2,
      "learning_rate": 8.90592499872884e-06,
      "logits/chosen": -1.9641724824905396,
      "logits/rejected": -1.9976900815963745,
      "logps/chosen": -167.5819091796875,
      "logps/rejected": -277.0826110839844,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5642932653427124,
      "rewards/margins": 9.902345657348633,
      "rewards/rejected": -10.466638565063477,
      "step": 1684
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.878523368724046e-06,
      "logits/chosen": -1.8020939826965332,
      "logits/rejected": -1.7672932147979736,
      "logps/chosen": -178.46359252929688,
      "logps/rejected": -260.63427734375,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.695149302482605,
      "rewards/margins": 8.129568099975586,
      "rewards/rejected": -9.82471752166748,
      "step": 1685
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.851154853614788e-06,
      "logits/chosen": -2.0805466175079346,
      "logits/rejected": -2.0776491165161133,
      "logps/chosen": -158.80641174316406,
      "logps/rejected": -236.09286499023438,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7098692655563354,
      "rewards/margins": 8.5169038772583,
      "rewards/rejected": -9.226773262023926,
      "step": 1686
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.823819509618364e-06,
      "logits/chosen": -1.8982199430465698,
      "logits/rejected": -1.8732504844665527,
      "logps/chosen": -167.55044555664062,
      "logps/rejected": -227.3775634765625,
      "loss": 0.0057,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0517865419387817,
      "rewards/margins": 7.997457504272461,
      "rewards/rejected": -9.049243927001953,
      "step": 1687
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.796517392883894e-06,
      "logits/chosen": -1.8792915344238281,
      "logits/rejected": -1.88819420337677,
      "logps/chosen": -158.40396118164062,
      "logps/rejected": -299.76507568359375,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1791374683380127,
      "rewards/margins": 10.26131534576416,
      "rewards/rejected": -12.440452575683594,
      "step": 1688
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.769248559492286e-06,
      "logits/chosen": -1.5488462448120117,
      "logits/rejected": -1.623557209968567,
      "logps/chosen": -156.12503051757812,
      "logps/rejected": -230.03541564941406,
      "loss": 0.0885,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4220478534698486,
      "rewards/margins": 6.3889875411987305,
      "rewards/rejected": -7.81103515625,
      "step": 1689
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.742013065456047e-06,
      "logits/chosen": -2.084272623062134,
      "logits/rejected": -2.063805103302002,
      "logps/chosen": -170.78964233398438,
      "logps/rejected": -270.0042419433594,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.35403430461883545,
      "rewards/margins": 10.892590522766113,
      "rewards/rejected": -10.538556098937988,
      "step": 1690
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.714810966719225e-06,
      "logits/chosen": -1.9889304637908936,
      "logits/rejected": -2.0050320625305176,
      "logps/chosen": -175.48451232910156,
      "logps/rejected": -276.85711669921875,
      "loss": 0.0873,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.2997756898403168,
      "rewards/margins": 10.429403305053711,
      "rewards/rejected": -10.729178428649902,
      "step": 1691
    },
    {
      "epoch": 2.21,
      "learning_rate": 8.687642319157279e-06,
      "logits/chosen": -1.5577396154403687,
      "logits/rejected": -1.6145566701889038,
      "logps/chosen": -175.64710998535156,
      "logps/rejected": -262.26422119140625,
      "loss": 0.0503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.697465181350708,
      "rewards/margins": 7.765927791595459,
      "rewards/rejected": -9.46339225769043,
      "step": 1692
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.660507178576907e-06,
      "logits/chosen": -1.9352643489837646,
      "logits/rejected": -2.1112453937530518,
      "logps/chosen": -135.61376953125,
      "logps/rejected": -229.64674377441406,
      "loss": 0.0879,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.214318037033081,
      "rewards/margins": 7.1794915199279785,
      "rewards/rejected": -8.39380931854248,
      "step": 1693
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.633405600716035e-06,
      "logits/chosen": -1.9938738346099854,
      "logits/rejected": -1.9770281314849854,
      "logps/chosen": -177.5645294189453,
      "logps/rejected": -246.0801544189453,
      "loss": 0.045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7230372428894043,
      "rewards/margins": 6.728675365447998,
      "rewards/rejected": -8.451711654663086,
      "step": 1694
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.606337641243634e-06,
      "logits/chosen": -1.775531530380249,
      "logits/rejected": -1.732622742652893,
      "logps/chosen": -161.1515350341797,
      "logps/rejected": -230.94967651367188,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1585237979888916,
      "rewards/margins": 7.616741180419922,
      "rewards/rejected": -9.77526569366455,
      "step": 1695
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.579303355759597e-06,
      "logits/chosen": -1.7641441822052002,
      "logits/rejected": -1.7365045547485352,
      "logps/chosen": -140.2285919189453,
      "logps/rejected": -255.93116760253906,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2101223468780518,
      "rewards/margins": 9.090911865234375,
      "rewards/rejected": -10.301034927368164,
      "step": 1696
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.552302799794675e-06,
      "logits/chosen": -1.6582105159759521,
      "logits/rejected": -1.6940668821334839,
      "logps/chosen": -149.7952423095703,
      "logps/rejected": -237.59933471679688,
      "loss": 0.1307,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1992766261100769,
      "rewards/margins": 9.18216323852539,
      "rewards/rejected": -9.381439208984375,
      "step": 1697
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.525336028810333e-06,
      "logits/chosen": -1.9155163764953613,
      "logits/rejected": -1.914123773574829,
      "logps/chosen": -194.36505126953125,
      "logps/rejected": -251.3768768310547,
      "loss": 0.0893,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1204029321670532,
      "rewards/margins": 5.926624774932861,
      "rewards/rejected": -7.047028064727783,
      "step": 1698
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.498403098198621e-06,
      "logits/chosen": -1.6718111038208008,
      "logits/rejected": -1.706322193145752,
      "logps/chosen": -147.09169006347656,
      "logps/rejected": -227.16058349609375,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.380272388458252,
      "rewards/margins": 7.894168376922607,
      "rewards/rejected": -9.27444076538086,
      "step": 1699
    },
    {
      "epoch": 2.22,
      "learning_rate": 8.471504063282082e-06,
      "logits/chosen": -2.019446849822998,
      "logits/rejected": -2.111304998397827,
      "logps/chosen": -176.03016662597656,
      "logps/rejected": -278.40802001953125,
      "loss": 0.0452,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6634742021560669,
      "rewards/margins": 10.115530967712402,
      "rewards/rejected": -10.779006004333496,
      "step": 1700
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.444638979313647e-06,
      "logits/chosen": -2.061818838119507,
      "logits/rejected": -1.9687267541885376,
      "logps/chosen": -149.9883575439453,
      "logps/rejected": -226.3201446533203,
      "loss": 0.0463,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.45260298252105713,
      "rewards/margins": 8.239148139953613,
      "rewards/rejected": -8.691750526428223,
      "step": 1701
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.417807901476513e-06,
      "logits/chosen": -1.8015291690826416,
      "logits/rejected": -1.8592723608016968,
      "logps/chosen": -157.38450622558594,
      "logps/rejected": -246.9696044921875,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6353341937065125,
      "rewards/margins": 9.038431167602539,
      "rewards/rejected": -9.673765182495117,
      "step": 1702
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.391010884884008e-06,
      "logits/chosen": -1.9042896032333374,
      "logits/rejected": -1.9888910055160522,
      "logps/chosen": -169.1690673828125,
      "logps/rejected": -270.6376647949219,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2796200513839722,
      "rewards/margins": 8.799640655517578,
      "rewards/rejected": -10.07926082611084,
      "step": 1703
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.364247984579487e-06,
      "logits/chosen": -1.759635329246521,
      "logits/rejected": -1.881662368774414,
      "logps/chosen": -148.27650451660156,
      "logps/rejected": -252.82656860351562,
      "loss": 0.1319,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3236020803451538,
      "rewards/margins": 7.425538063049316,
      "rewards/rejected": -8.749139785766602,
      "step": 1704
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.337519255536259e-06,
      "logits/chosen": -1.9671367406845093,
      "logits/rejected": -1.973100185394287,
      "logps/chosen": -159.25816345214844,
      "logps/rejected": -246.9493408203125,
      "loss": 0.0484,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.737236499786377,
      "rewards/margins": 7.744680881500244,
      "rewards/rejected": -9.481916427612305,
      "step": 1705
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.310824752657426e-06,
      "logits/chosen": -1.8452237844467163,
      "logits/rejected": -1.8434114456176758,
      "logps/chosen": -186.22930908203125,
      "logps/rejected": -261.36175537109375,
      "loss": 0.0031,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4017562866210938,
      "rewards/margins": 7.967874526977539,
      "rewards/rejected": -9.36962890625,
      "step": 1706
    },
    {
      "epoch": 2.23,
      "learning_rate": 8.284164530775776e-06,
      "logits/chosen": -1.7252148389816284,
      "logits/rejected": -1.7483556270599365,
      "logps/chosen": -181.51739501953125,
      "logps/rejected": -255.73422241210938,
      "loss": 0.0891,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4551174640655518,
      "rewards/margins": 7.139329433441162,
      "rewards/rejected": -8.594447135925293,
      "step": 1707
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.257538644653695e-06,
      "logits/chosen": -2.0242156982421875,
      "logits/rejected": -2.0060341358184814,
      "logps/chosen": -172.74961853027344,
      "logps/rejected": -230.2306671142578,
      "loss": 0.0484,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7689175605773926,
      "rewards/margins": 7.378180027008057,
      "rewards/rejected": -9.14709758758545,
      "step": 1708
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.230947148983056e-06,
      "logits/chosen": -1.7818617820739746,
      "logits/rejected": -1.8929425477981567,
      "logps/chosen": -155.7158660888672,
      "logps/rejected": -249.40658569335938,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0769742727279663,
      "rewards/margins": 8.7205228805542,
      "rewards/rejected": -9.797496795654297,
      "step": 1709
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.20439009838504e-06,
      "logits/chosen": -1.687815546989441,
      "logits/rejected": -1.6670013666152954,
      "logps/chosen": -185.51010131835938,
      "logps/rejected": -251.84873962402344,
      "loss": 0.0939,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6286787986755371,
      "rewards/margins": 7.691103935241699,
      "rewards/rejected": -8.319782257080078,
      "step": 1710
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.177867547410117e-06,
      "logits/chosen": -2.043875217437744,
      "logits/rejected": -2.0258073806762695,
      "logps/chosen": -170.44403076171875,
      "logps/rejected": -245.239501953125,
      "loss": 0.0493,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8934284448623657,
      "rewards/margins": 8.273063659667969,
      "rewards/rejected": -9.16649055480957,
      "step": 1711
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.151379550537894e-06,
      "logits/chosen": -1.8785483837127686,
      "logits/rejected": -1.9934923648834229,
      "logps/chosen": -143.48031616210938,
      "logps/rejected": -245.91107177734375,
      "loss": 0.009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9724593758583069,
      "rewards/margins": 8.94121265411377,
      "rewards/rejected": -9.913671493530273,
      "step": 1712
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.124926162176972e-06,
      "logits/chosen": -2.0352823734283447,
      "logits/rejected": -2.0337705612182617,
      "logps/chosen": -149.98182678222656,
      "logps/rejected": -215.06735229492188,
      "loss": 0.0876,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.34417861700057983,
      "rewards/margins": 7.40322208404541,
      "rewards/rejected": -7.74739933013916,
      "step": 1713
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.09850743666489e-06,
      "logits/chosen": -1.7941476106643677,
      "logits/rejected": -1.8420958518981934,
      "logps/chosen": -137.38316345214844,
      "logps/rejected": -223.0071563720703,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.967444896697998,
      "rewards/margins": 8.241368293762207,
      "rewards/rejected": -9.208812713623047,
      "step": 1714
    },
    {
      "epoch": 2.24,
      "learning_rate": 8.072123428267966e-06,
      "logits/chosen": -1.7162954807281494,
      "logits/rejected": -1.6305172443389893,
      "logps/chosen": -150.4332275390625,
      "logps/rejected": -224.67185974121094,
      "loss": 0.0487,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9232409000396729,
      "rewards/margins": 7.295369625091553,
      "rewards/rejected": -8.218610763549805,
      "step": 1715
    },
    {
      "epoch": 2.25,
      "learning_rate": 8.045774191181229e-06,
      "logits/chosen": -1.9025371074676514,
      "logits/rejected": -1.8475695848464966,
      "logps/chosen": -186.20428466796875,
      "logps/rejected": -258.87548828125,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.444622278213501,
      "rewards/margins": 8.203651428222656,
      "rewards/rejected": -10.648273468017578,
      "step": 1716
    },
    {
      "epoch": 2.25,
      "learning_rate": 8.01945977952826e-06,
      "logits/chosen": -2.009754180908203,
      "logits/rejected": -1.9731206893920898,
      "logps/chosen": -172.5438995361328,
      "logps/rejected": -284.9532775878906,
      "loss": 0.0464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.577627182006836,
      "rewards/margins": 9.668525695800781,
      "rewards/rejected": -11.246153831481934,
      "step": 1717
    },
    {
      "epoch": 2.25,
      "learning_rate": 7.993180247361117e-06,
      "logits/chosen": -1.9226957559585571,
      "logits/rejected": -1.8754510879516602,
      "logps/chosen": -193.69459533691406,
      "logps/rejected": -255.15484619140625,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.869742751121521,
      "rewards/margins": 7.790713310241699,
      "rewards/rejected": -9.660455703735352,
      "step": 1718
    },
    {
      "epoch": 2.25,
      "learning_rate": 7.966935648660229e-06,
      "logits/chosen": -1.7804970741271973,
      "logits/rejected": -1.6659536361694336,
      "logps/chosen": -152.5810089111328,
      "logps/rejected": -218.44825744628906,
      "loss": 0.007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.19271478056907654,
      "rewards/margins": 8.395965576171875,
      "rewards/rejected": -8.5886812210083,
      "step": 1719
    },
    {
      "epoch": 2.25,
      "learning_rate": 7.940726037334237e-06,
      "logits/chosen": -1.8840152025222778,
      "logits/rejected": -1.8936865329742432,
      "logps/chosen": -139.13388061523438,
      "logps/rejected": -229.00100708007812,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8906530141830444,
      "rewards/margins": 8.860274314880371,
      "rewards/rejected": -9.750926971435547,
      "step": 1720
    },
    {
      "epoch": 2.25,
      "learning_rate": 7.914551467219928e-06,
      "logits/chosen": -1.9961819648742676,
      "logits/rejected": -2.0198922157287598,
      "logps/chosen": -156.68238830566406,
      "logps/rejected": -231.15621948242188,
      "loss": 0.0468,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.3314359188079834,
      "rewards/margins": 6.614280700683594,
      "rewards/rejected": -8.94571590423584,
      "step": 1721
    },
    {
      "epoch": 2.25,
      "learning_rate": 7.88841199208212e-06,
      "logits/chosen": -1.9082878828048706,
      "logits/rejected": -1.9189866781234741,
      "logps/chosen": -145.55975341796875,
      "logps/rejected": -203.27590942382812,
      "loss": 0.1332,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.9775505065917969,
      "rewards/margins": 6.369935035705566,
      "rewards/rejected": -7.347485542297363,
      "step": 1722
    },
    {
      "epoch": 2.25,
      "learning_rate": 7.862307665613543e-06,
      "logits/chosen": -1.7508575916290283,
      "logits/rejected": -1.631354570388794,
      "logps/chosen": -182.1580810546875,
      "logps/rejected": -252.74581909179688,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9741643071174622,
      "rewards/margins": 8.724651336669922,
      "rewards/rejected": -9.698816299438477,
      "step": 1723
    },
    {
      "epoch": 2.26,
      "learning_rate": 7.836238541434709e-06,
      "logits/chosen": -1.912994384765625,
      "logits/rejected": -1.9540929794311523,
      "logps/chosen": -176.66160583496094,
      "logps/rejected": -262.5517883300781,
      "loss": 0.0479,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9405203461647034,
      "rewards/margins": 7.829130172729492,
      "rewards/rejected": -8.769651412963867,
      "step": 1724
    },
    {
      "epoch": 2.26,
      "learning_rate": 7.810204673093848e-06,
      "logits/chosen": -1.8203470706939697,
      "logits/rejected": -1.8752679824829102,
      "logps/chosen": -170.28573608398438,
      "logps/rejected": -249.3515167236328,
      "loss": 0.0038,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0952033996582031,
      "rewards/margins": 7.94855260848999,
      "rewards/rejected": -9.043755531311035,
      "step": 1725
    },
    {
      "epoch": 2.26,
      "learning_rate": 7.784206114066753e-06,
      "logits/chosen": -1.9490711688995361,
      "logits/rejected": -1.9893689155578613,
      "logps/chosen": -170.83937072753906,
      "logps/rejected": -294.0124206542969,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9843921065330505,
      "rewards/margins": 11.237875938415527,
      "rewards/rejected": -12.222267150878906,
      "step": 1726
    },
    {
      "epoch": 2.26,
      "learning_rate": 7.758242917756683e-06,
      "logits/chosen": -2.076385021209717,
      "logits/rejected": -2.077730655670166,
      "logps/chosen": -163.35780334472656,
      "logps/rejected": -266.3843994140625,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17842397093772888,
      "rewards/margins": 10.363500595092773,
      "rewards/rejected": -10.541924476623535,
      "step": 1727
    },
    {
      "epoch": 2.26,
      "learning_rate": 7.732315137494277e-06,
      "logits/chosen": -1.915196418762207,
      "logits/rejected": -1.9771504402160645,
      "logps/chosen": -164.85789489746094,
      "logps/rejected": -275.79949951171875,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9115972518920898,
      "rewards/margins": 8.350415229797363,
      "rewards/rejected": -9.262012481689453,
      "step": 1728
    },
    {
      "epoch": 2.26,
      "learning_rate": 7.706422826537435e-06,
      "logits/chosen": -1.463667869567871,
      "logits/rejected": -1.5436584949493408,
      "logps/chosen": -186.6462860107422,
      "logps/rejected": -286.0638732910156,
      "loss": 0.0447,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.792382836341858,
      "rewards/margins": 8.272198677062988,
      "rewards/rejected": -10.064580917358398,
      "step": 1729
    },
    {
      "epoch": 2.26,
      "learning_rate": 7.680566038071157e-06,
      "logits/chosen": -1.7313063144683838,
      "logits/rejected": -1.7808032035827637,
      "logps/chosen": -238.25758361816406,
      "logps/rejected": -316.22900390625,
      "loss": 0.131,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5334932804107666,
      "rewards/margins": 7.321683406829834,
      "rewards/rejected": -8.855175971984863,
      "step": 1730
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.654744825207527e-06,
      "logits/chosen": -1.7623226642608643,
      "logits/rejected": -1.7620604038238525,
      "logps/chosen": -178.7823028564453,
      "logps/rejected": -237.9049835205078,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9732635021209717,
      "rewards/margins": 7.484884262084961,
      "rewards/rejected": -9.458147048950195,
      "step": 1731
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.628959240985514e-06,
      "logits/chosen": -1.7217204570770264,
      "logits/rejected": -1.691422700881958,
      "logps/chosen": -205.74200439453125,
      "logps/rejected": -308.8719177246094,
      "loss": 0.0437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6251065731048584,
      "rewards/margins": 9.73115062713623,
      "rewards/rejected": -10.356256484985352,
      "step": 1732
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.6032093383709345e-06,
      "logits/chosen": -1.8879531621932983,
      "logits/rejected": -1.9387571811676025,
      "logps/chosen": -153.4672393798828,
      "logps/rejected": -239.40585327148438,
      "loss": 0.0489,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2591915130615234,
      "rewards/margins": 8.240959167480469,
      "rewards/rejected": -9.500150680541992,
      "step": 1733
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.57749517025628e-06,
      "logits/chosen": -1.9365811347961426,
      "logits/rejected": -2.017869472503662,
      "logps/chosen": -158.4349365234375,
      "logps/rejected": -267.27008056640625,
      "loss": 0.0714,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6609711050987244,
      "rewards/margins": 9.28534984588623,
      "rewards/rejected": -9.946321487426758,
      "step": 1734
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.551816789460664e-06,
      "logits/chosen": -1.8257447481155396,
      "logits/rejected": -1.760074257850647,
      "logps/chosen": -202.08291625976562,
      "logps/rejected": -283.73846435546875,
      "loss": 0.0455,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.930687665939331,
      "rewards/margins": 8.253698348999023,
      "rewards/rejected": -9.184385299682617,
      "step": 1735
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.5261742487297e-06,
      "logits/chosen": -1.9845526218414307,
      "logits/rejected": -1.9955781698226929,
      "logps/chosen": -177.87852478027344,
      "logps/rejected": -234.1166229248047,
      "loss": 0.1923,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9085915088653564,
      "rewards/margins": 5.927431106567383,
      "rewards/rejected": -7.83602237701416,
      "step": 1736
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.5005676007353364e-06,
      "logits/chosen": -1.9292917251586914,
      "logits/rejected": -1.9607986211776733,
      "logps/chosen": -205.56668090820312,
      "logps/rejected": -265.26312255859375,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5743094682693481,
      "rewards/margins": 7.02839469909668,
      "rewards/rejected": -7.602704048156738,
      "step": 1737
    },
    {
      "epoch": 2.27,
      "learning_rate": 7.4749968980758365e-06,
      "logits/chosen": -1.7907233238220215,
      "logits/rejected": -1.7654228210449219,
      "logps/chosen": -182.60528564453125,
      "logps/rejected": -267.1372375488281,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6779470443725586,
      "rewards/margins": 8.459836959838867,
      "rewards/rejected": -10.137784004211426,
      "step": 1738
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.449462193275628e-06,
      "logits/chosen": -1.7250425815582275,
      "logits/rejected": -1.7074041366577148,
      "logps/chosen": -191.56973266601562,
      "logps/rejected": -254.9542694091797,
      "loss": 0.0883,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3050459623336792,
      "rewards/margins": 7.720423698425293,
      "rewards/rejected": -9.025468826293945,
      "step": 1739
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.4239635387851615e-06,
      "logits/chosen": -1.8569387197494507,
      "logits/rejected": -1.8807289600372314,
      "logps/chosen": -167.26425170898438,
      "logps/rejected": -263.0946960449219,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9089138507843018,
      "rewards/margins": 9.107555389404297,
      "rewards/rejected": -10.016469955444336,
      "step": 1740
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.398500986980877e-06,
      "logits/chosen": -1.6602556705474854,
      "logits/rejected": -1.664499044418335,
      "logps/chosen": -188.3311767578125,
      "logps/rejected": -297.74603271484375,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.38895514607429504,
      "rewards/margins": 10.47437858581543,
      "rewards/rejected": -10.863333702087402,
      "step": 1741
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.373074590165041e-06,
      "logits/chosen": -1.8295729160308838,
      "logits/rejected": -1.9674129486083984,
      "logps/chosen": -171.72146606445312,
      "logps/rejected": -271.66802978515625,
      "loss": 0.1134,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.1419577598571777,
      "rewards/margins": 7.18996524810791,
      "rewards/rejected": -9.33192253112793,
      "step": 1742
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.347684400565646e-06,
      "logits/chosen": -1.536153793334961,
      "logits/rejected": -1.5268285274505615,
      "logps/chosen": -182.3928985595703,
      "logps/rejected": -305.0445556640625,
      "loss": 0.0573,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.9637885093688965,
      "rewards/margins": 9.35505485534668,
      "rewards/rejected": -11.318843841552734,
      "step": 1743
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.3223304703363135e-06,
      "logits/chosen": -1.7593497037887573,
      "logits/rejected": -1.8221255540847778,
      "logps/chosen": -160.93511962890625,
      "logps/rejected": -233.61209106445312,
      "loss": 0.089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4239659309387207,
      "rewards/margins": 6.549699783325195,
      "rewards/rejected": -7.973665237426758,
      "step": 1744
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.297012851556198e-06,
      "logits/chosen": -1.8462677001953125,
      "logits/rejected": -1.8629298210144043,
      "logps/chosen": -168.32041931152344,
      "logps/rejected": -252.20001220703125,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6456645727157593,
      "rewards/margins": 8.742741584777832,
      "rewards/rejected": -10.388405799865723,
      "step": 1745
    },
    {
      "epoch": 2.28,
      "learning_rate": 7.271731596229864e-06,
      "logits/chosen": -1.8396629095077515,
      "logits/rejected": -1.851818323135376,
      "logps/chosen": -184.07437133789062,
      "logps/rejected": -269.8797607421875,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4669995307922363,
      "rewards/margins": 8.512470245361328,
      "rewards/rejected": -9.979469299316406,
      "step": 1746
    },
    {
      "epoch": 2.29,
      "learning_rate": 7.2464867562871745e-06,
      "logits/chosen": -1.8737417459487915,
      "logits/rejected": -1.847265362739563,
      "logps/chosen": -186.4523162841797,
      "logps/rejected": -275.9980163574219,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4697492122650146,
      "rewards/margins": 8.279195785522461,
      "rewards/rejected": -9.748944282531738,
      "step": 1747
    },
    {
      "epoch": 2.29,
      "learning_rate": 7.221278383583185e-06,
      "logits/chosen": -1.6506069898605347,
      "logits/rejected": -1.6418254375457764,
      "logps/chosen": -173.87623596191406,
      "logps/rejected": -282.06591796875,
      "loss": 0.0036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3546056747436523,
      "rewards/margins": 9.797117233276367,
      "rewards/rejected": -11.15172290802002,
      "step": 1748
    },
    {
      "epoch": 2.29,
      "learning_rate": 7.1961065298980666e-06,
      "logits/chosen": -1.8045096397399902,
      "logits/rejected": -1.8286653757095337,
      "logps/chosen": -159.36940002441406,
      "logps/rejected": -254.413818359375,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2125332355499268,
      "rewards/margins": 8.592373847961426,
      "rewards/rejected": -9.80490779876709,
      "step": 1749
    },
    {
      "epoch": 2.29,
      "learning_rate": 7.170971246936966e-06,
      "logits/chosen": -2.080958366394043,
      "logits/rejected": -2.0719833374023438,
      "logps/chosen": -183.59608459472656,
      "logps/rejected": -266.4599609375,
      "loss": 0.0451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3536672592163086,
      "rewards/margins": 8.073270797729492,
      "rewards/rejected": -9.4269380569458,
      "step": 1750
    },
    {
      "epoch": 2.29,
      "learning_rate": 7.145872586329902e-06,
      "logits/chosen": -1.9525196552276611,
      "logits/rejected": -1.8959940671920776,
      "logps/chosen": -162.29661560058594,
      "logps/rejected": -264.79229736328125,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7748004198074341,
      "rewards/margins": 10.172969818115234,
      "rewards/rejected": -10.947770118713379,
      "step": 1751
    },
    {
      "epoch": 2.29,
      "learning_rate": 7.12081059963168e-06,
      "logits/chosen": -1.8730671405792236,
      "logits/rejected": -1.8455735445022583,
      "logps/chosen": -155.40843200683594,
      "logps/rejected": -237.49789428710938,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4315401315689087,
      "rewards/margins": 8.454864501953125,
      "rewards/rejected": -9.886404037475586,
      "step": 1752
    },
    {
      "epoch": 2.29,
      "learning_rate": 7.095785338321787e-06,
      "logits/chosen": -1.9028881788253784,
      "logits/rejected": -1.90817129611969,
      "logps/chosen": -169.12850952148438,
      "logps/rejected": -255.319091796875,
      "loss": 0.0885,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.932592749595642,
      "rewards/margins": 8.128854751586914,
      "rewards/rejected": -10.061447143554688,
      "step": 1753
    },
    {
      "epoch": 2.3,
      "learning_rate": 7.070796853804221e-06,
      "logits/chosen": -1.9785782098770142,
      "logits/rejected": -1.9396729469299316,
      "logps/chosen": -147.90042114257812,
      "logps/rejected": -238.70632934570312,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1804263591766357,
      "rewards/margins": 9.65223503112793,
      "rewards/rejected": -10.832659721374512,
      "step": 1754
    },
    {
      "epoch": 2.3,
      "learning_rate": 7.045845197407494e-06,
      "logits/chosen": -1.7900846004486084,
      "logits/rejected": -1.8115050792694092,
      "logps/chosen": -180.84201049804688,
      "logps/rejected": -276.5029296875,
      "loss": 0.0464,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8495785593986511,
      "rewards/margins": 9.0549898147583,
      "rewards/rejected": -9.904568672180176,
      "step": 1755
    },
    {
      "epoch": 2.3,
      "learning_rate": 7.02093042038445e-06,
      "logits/chosen": -1.4954321384429932,
      "logits/rejected": -1.5247416496276855,
      "logps/chosen": -147.614990234375,
      "logps/rejected": -254.51893615722656,
      "loss": 0.0048,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.232337474822998,
      "rewards/margins": 8.843574523925781,
      "rewards/rejected": -11.075912475585938,
      "step": 1756
    },
    {
      "epoch": 2.3,
      "learning_rate": 6.996052573912163e-06,
      "logits/chosen": -1.6610817909240723,
      "logits/rejected": -1.6910808086395264,
      "logps/chosen": -148.6898193359375,
      "logps/rejected": -243.25047302246094,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6532607078552246,
      "rewards/margins": 9.277359962463379,
      "rewards/rejected": -9.930620193481445,
      "step": 1757
    },
    {
      "epoch": 2.3,
      "learning_rate": 6.971211709091882e-06,
      "logits/chosen": -1.8176690340042114,
      "logits/rejected": -1.844098448753357,
      "logps/chosen": -151.39456176757812,
      "logps/rejected": -223.08865356445312,
      "loss": 0.0887,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7704759836196899,
      "rewards/margins": 8.49703598022461,
      "rewards/rejected": -9.267512321472168,
      "step": 1758
    },
    {
      "epoch": 2.3,
      "learning_rate": 6.946407876948854e-06,
      "logits/chosen": -2.0205297470092773,
      "logits/rejected": -2.0090224742889404,
      "logps/chosen": -161.44943237304688,
      "logps/rejected": -249.68597412109375,
      "loss": 0.0501,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.407362461090088,
      "rewards/margins": 9.24038028717041,
      "rewards/rejected": -10.647741317749023,
      "step": 1759
    },
    {
      "epoch": 2.3,
      "learning_rate": 6.921641128432299e-06,
      "logits/chosen": -1.6458789110183716,
      "logits/rejected": -1.6404844522476196,
      "logps/chosen": -177.78976440429688,
      "logps/rejected": -286.12872314453125,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1124839782714844,
      "rewards/margins": 8.5626859664917,
      "rewards/rejected": -9.675168991088867,
      "step": 1760
    },
    {
      "epoch": 2.3,
      "learning_rate": 6.896911514415219e-06,
      "logits/chosen": -1.6858186721801758,
      "logits/rejected": -1.7197470664978027,
      "logps/chosen": -163.48587036132812,
      "logps/rejected": -272.69561767578125,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.050740361213684,
      "rewards/margins": 9.518424034118652,
      "rewards/rejected": -10.569164276123047,
      "step": 1761
    },
    {
      "epoch": 2.31,
      "learning_rate": 6.872219085694376e-06,
      "logits/chosen": -2.0083394050598145,
      "logits/rejected": -1.93034827709198,
      "logps/chosen": -216.573486328125,
      "logps/rejected": -264.935546875,
      "loss": 0.0514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.46710824966430664,
      "rewards/margins": 7.933774948120117,
      "rewards/rejected": -8.400882720947266,
      "step": 1762
    },
    {
      "epoch": 2.31,
      "learning_rate": 6.8475638929901385e-06,
      "logits/chosen": -1.5732144117355347,
      "logits/rejected": -1.7223548889160156,
      "logps/chosen": -196.32293701171875,
      "logps/rejected": -291.34674072265625,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.61316978931427,
      "rewards/margins": 9.078568458557129,
      "rewards/rejected": -9.691740036010742,
      "step": 1763
    },
    {
      "epoch": 2.31,
      "learning_rate": 6.822945986946386e-06,
      "logits/chosen": -1.7464098930358887,
      "logits/rejected": -1.808815360069275,
      "logps/chosen": -208.31961059570312,
      "logps/rejected": -322.5469665527344,
      "loss": 0.0871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1374528408050537,
      "rewards/margins": 9.0220365524292,
      "rewards/rejected": -11.159488677978516,
      "step": 1764
    },
    {
      "epoch": 2.31,
      "learning_rate": 6.798365418130395e-06,
      "logits/chosen": -1.8705166578292847,
      "logits/rejected": -1.9167060852050781,
      "logps/chosen": -165.36386108398438,
      "logps/rejected": -270.8473205566406,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.063851237297058,
      "rewards/margins": 9.956506729125977,
      "rewards/rejected": -11.020358085632324,
      "step": 1765
    },
    {
      "epoch": 2.31,
      "learning_rate": 6.773822237032779e-06,
      "logits/chosen": -1.6601667404174805,
      "logits/rejected": -1.7148581743240356,
      "logps/chosen": -165.89556884765625,
      "logps/rejected": -290.2835693359375,
      "loss": 0.087,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2235369682312012,
      "rewards/margins": 11.60428524017334,
      "rewards/rejected": -12.8278226852417,
      "step": 1766
    },
    {
      "epoch": 2.31,
      "learning_rate": 6.74931649406732e-06,
      "logits/chosen": -2.0220162868499756,
      "logits/rejected": -2.009742259979248,
      "logps/chosen": -149.424560546875,
      "logps/rejected": -216.11825561523438,
      "loss": 0.0554,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7101175785064697,
      "rewards/margins": 7.009714126586914,
      "rewards/rejected": -8.719831466674805,
      "step": 1767
    },
    {
      "epoch": 2.31,
      "learning_rate": 6.724848239570927e-06,
      "logits/chosen": -1.777890682220459,
      "logits/rejected": -1.8965660333633423,
      "logps/chosen": -158.99940490722656,
      "logps/rejected": -273.4164123535156,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5410434007644653,
      "rewards/margins": 10.910515785217285,
      "rewards/rejected": -11.451560020446777,
      "step": 1768
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.700417523803498e-06,
      "logits/chosen": -2.099468469619751,
      "logits/rejected": -2.1086387634277344,
      "logps/chosen": -164.02256774902344,
      "logps/rejected": -274.2021789550781,
      "loss": 0.0034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.30260759592056274,
      "rewards/margins": 10.075713157653809,
      "rewards/rejected": -10.378321647644043,
      "step": 1769
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.6760243969478105e-06,
      "logits/chosen": -1.7251557111740112,
      "logits/rejected": -1.7839467525482178,
      "logps/chosen": -160.01219177246094,
      "logps/rejected": -280.53533935546875,
      "loss": 0.0005,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0813661813735962,
      "rewards/margins": 10.289020538330078,
      "rewards/rejected": -11.370386123657227,
      "step": 1770
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.651668909109435e-06,
      "logits/chosen": -1.7191877365112305,
      "logits/rejected": -1.7124154567718506,
      "logps/chosen": -145.8040313720703,
      "logps/rejected": -263.005859375,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2123868465423584,
      "rewards/margins": 10.001996040344238,
      "rewards/rejected": -11.21438217163086,
      "step": 1771
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.627351110316635e-06,
      "logits/chosen": -1.8327285051345825,
      "logits/rejected": -1.8456400632858276,
      "logps/chosen": -221.40966796875,
      "logps/rejected": -310.8492431640625,
      "loss": 0.0882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1594538688659668,
      "rewards/margins": 8.74454116821289,
      "rewards/rejected": -9.9039945602417,
      "step": 1772
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.603071050520262e-06,
      "logits/chosen": -1.9578769207000732,
      "logits/rejected": -2.036078453063965,
      "logps/chosen": -165.41470336914062,
      "logps/rejected": -264.12054443359375,
      "loss": 0.0973,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.067358136177063,
      "rewards/margins": 9.457319259643555,
      "rewards/rejected": -10.524678230285645,
      "step": 1773
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.578828779593632e-06,
      "logits/chosen": -1.8775224685668945,
      "logits/rejected": -1.8545790910720825,
      "logps/chosen": -184.04299926757812,
      "logps/rejected": -275.00341796875,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1954989433288574,
      "rewards/margins": 9.004657745361328,
      "rewards/rejected": -11.200156211853027,
      "step": 1774
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.554624347332458e-06,
      "logits/chosen": -2.020444869995117,
      "logits/rejected": -2.088294506072998,
      "logps/chosen": -192.11737060546875,
      "logps/rejected": -286.9687805175781,
      "loss": 0.0009,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0175933837890625,
      "rewards/margins": 9.734195709228516,
      "rewards/rejected": -10.751787185668945,
      "step": 1775
    },
    {
      "epoch": 2.32,
      "learning_rate": 6.530457803454707e-06,
      "logits/chosen": -1.8145325183868408,
      "logits/rejected": -1.9068529605865479,
      "logps/chosen": -189.66525268554688,
      "logps/rejected": -279.6603088378906,
      "loss": 0.0442,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9589489698410034,
      "rewards/margins": 9.191974639892578,
      "rewards/rejected": -11.150923728942871,
      "step": 1776
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.5063291976005445e-06,
      "logits/chosen": -1.939195990562439,
      "logits/rejected": -1.92375910282135,
      "logps/chosen": -161.25393676757812,
      "logps/rejected": -272.2678527832031,
      "loss": 0.0007,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5990134477615356,
      "rewards/margins": 9.84693431854248,
      "rewards/rejected": -10.445947647094727,
      "step": 1777
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.482238579332184e-06,
      "logits/chosen": -1.9336549043655396,
      "logits/rejected": -1.9684922695159912,
      "logps/chosen": -164.7434539794922,
      "logps/rejected": -245.27560424804688,
      "loss": 0.047,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5549672842025757,
      "rewards/margins": 8.15001392364502,
      "rewards/rejected": -8.704980850219727,
      "step": 1778
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.458185998133828e-06,
      "logits/chosen": -1.8066821098327637,
      "logits/rejected": -1.7920650243759155,
      "logps/chosen": -180.28982543945312,
      "logps/rejected": -269.5885925292969,
      "loss": 0.0977,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.9195547699928284,
      "rewards/margins": 9.124629020690918,
      "rewards/rejected": -10.044183731079102,
      "step": 1779
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.434171503411557e-06,
      "logits/chosen": -1.6550133228302002,
      "logits/rejected": -1.7255570888519287,
      "logps/chosen": -185.69833374023438,
      "logps/rejected": -285.2115478515625,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5277138948440552,
      "rewards/margins": 8.827119827270508,
      "rewards/rejected": -10.35483455657959,
      "step": 1780
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.4101951444931725e-06,
      "logits/chosen": -1.83838951587677,
      "logits/rejected": -1.7970024347305298,
      "logps/chosen": -156.9870147705078,
      "logps/rejected": -250.90350341796875,
      "loss": 0.0448,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2961896657943726,
      "rewards/margins": 8.59891414642334,
      "rewards/rejected": -9.89510440826416,
      "step": 1781
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.386256970628185e-06,
      "logits/chosen": -1.858890414237976,
      "logits/rejected": -1.8017386198043823,
      "logps/chosen": -166.10833740234375,
      "logps/rejected": -254.23463439941406,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9960359334945679,
      "rewards/margins": 9.699010848999023,
      "rewards/rejected": -10.695047378540039,
      "step": 1782
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.362357030987667e-06,
      "logits/chosen": -1.9099041223526,
      "logits/rejected": -1.996301531791687,
      "logps/chosen": -149.42738342285156,
      "logps/rejected": -262.5460205078125,
      "loss": 0.0027,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6086513996124268,
      "rewards/margins": 8.800227165222168,
      "rewards/rejected": -10.408878326416016,
      "step": 1783
    },
    {
      "epoch": 2.33,
      "learning_rate": 6.338495374664127e-06,
      "logits/chosen": -1.9749311208724976,
      "logits/rejected": -2.0221550464630127,
      "logps/chosen": -156.06040954589844,
      "logps/rejected": -268.9734802246094,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9908095598220825,
      "rewards/margins": 9.94205379486084,
      "rewards/rejected": -10.932862281799316,
      "step": 1784
    },
    {
      "epoch": 2.34,
      "learning_rate": 6.314672050671461e-06,
      "logits/chosen": -1.767091155052185,
      "logits/rejected": -1.7567050457000732,
      "logps/chosen": -220.52542114257812,
      "logps/rejected": -285.19976806640625,
      "loss": 0.1747,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.829897165298462,
      "rewards/margins": 6.9384236335754395,
      "rewards/rejected": -8.768320083618164,
      "step": 1785
    },
    {
      "epoch": 2.34,
      "learning_rate": 6.290887107944826e-06,
      "logits/chosen": -1.8326233625411987,
      "logits/rejected": -1.8856908082962036,
      "logps/chosen": -182.89964294433594,
      "logps/rejected": -261.6730041503906,
      "loss": 0.0445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4302911758422852,
      "rewards/margins": 8.534648895263672,
      "rewards/rejected": -9.96493911743164,
      "step": 1786
    },
    {
      "epoch": 2.34,
      "learning_rate": 6.267140595340529e-06,
      "logits/chosen": -1.7465630769729614,
      "logits/rejected": -1.8231840133666992,
      "logps/chosen": -159.20132446289062,
      "logps/rejected": -272.36639404296875,
      "loss": 0.0043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.40208858251571655,
      "rewards/margins": 10.401426315307617,
      "rewards/rejected": -10.80351448059082,
      "step": 1787
    },
    {
      "epoch": 2.34,
      "learning_rate": 6.243432561635934e-06,
      "logits/chosen": -2.035583019256592,
      "logits/rejected": -2.0303378105163574,
      "logps/chosen": -164.10055541992188,
      "logps/rejected": -236.49407958984375,
      "loss": 0.0502,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.430525302886963,
      "rewards/margins": 7.718084335327148,
      "rewards/rejected": -9.148609161376953,
      "step": 1788
    },
    {
      "epoch": 2.34,
      "learning_rate": 6.219763055529384e-06,
      "logits/chosen": -1.8954731225967407,
      "logits/rejected": -1.8607579469680786,
      "logps/chosen": -150.72206115722656,
      "logps/rejected": -233.33261108398438,
      "loss": 0.0469,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0046842098236084,
      "rewards/margins": 7.844903945922852,
      "rewards/rejected": -9.849588394165039,
      "step": 1789
    },
    {
      "epoch": 2.34,
      "learning_rate": 6.1961321256400836e-06,
      "logits/chosen": -2.018836736679077,
      "logits/rejected": -2.0137195587158203,
      "logps/chosen": -171.15249633789062,
      "logps/rejected": -244.03517150878906,
      "loss": 0.0882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.222543716430664,
      "rewards/margins": 6.9049391746521,
      "rewards/rejected": -9.127482414245605,
      "step": 1790
    },
    {
      "epoch": 2.34,
      "learning_rate": 6.172539820507977e-06,
      "logits/chosen": -1.7762372493743896,
      "logits/rejected": -1.7680360078811646,
      "logps/chosen": -257.6258239746094,
      "logps/rejected": -364.79766845703125,
      "loss": 0.0637,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0148355960845947,
      "rewards/margins": 8.235854148864746,
      "rewards/rejected": -9.250690460205078,
      "step": 1791
    },
    {
      "epoch": 2.35,
      "learning_rate": 6.1489861885936805e-06,
      "logits/chosen": -1.5569305419921875,
      "logits/rejected": -1.59669029712677,
      "logps/chosen": -166.54302978515625,
      "logps/rejected": -273.8870849609375,
      "loss": 0.0453,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5444955825805664,
      "rewards/margins": 9.311293601989746,
      "rewards/rejected": -9.855789184570312,
      "step": 1792
    },
    {
      "epoch": 2.35,
      "learning_rate": 6.125471278278378e-06,
      "logits/chosen": -2.0254414081573486,
      "logits/rejected": -2.0716552734375,
      "logps/chosen": -153.61468505859375,
      "logps/rejected": -246.28028869628906,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.36680126190185547,
      "rewards/margins": 8.961858749389648,
      "rewards/rejected": -9.328660011291504,
      "step": 1793
    },
    {
      "epoch": 2.35,
      "learning_rate": 6.101995137863717e-06,
      "logits/chosen": -2.088609457015991,
      "logits/rejected": -2.156895160675049,
      "logps/chosen": -249.47723388671875,
      "logps/rejected": -326.8277587890625,
      "loss": 0.1746,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.0209801197052,
      "rewards/margins": 6.242717742919922,
      "rewards/rejected": -8.26369857788086,
      "step": 1794
    },
    {
      "epoch": 2.35,
      "learning_rate": 6.078557815571692e-06,
      "logits/chosen": -1.941967248916626,
      "logits/rejected": -1.9487855434417725,
      "logps/chosen": -145.223388671875,
      "logps/rejected": -225.42185974121094,
      "loss": 0.0604,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9959244132041931,
      "rewards/margins": 7.582269668579102,
      "rewards/rejected": -8.578194618225098,
      "step": 1795
    },
    {
      "epoch": 2.35,
      "learning_rate": 6.055159359544579e-06,
      "logits/chosen": -1.9348758459091187,
      "logits/rejected": -1.9461047649383545,
      "logps/chosen": -193.15744018554688,
      "logps/rejected": -275.7863464355469,
      "loss": 0.0886,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5319037437438965,
      "rewards/margins": 8.009862899780273,
      "rewards/rejected": -9.541766166687012,
      "step": 1796
    },
    {
      "epoch": 2.35,
      "learning_rate": 6.03179981784483e-06,
      "logits/chosen": -1.592857837677002,
      "logits/rejected": -1.614403486251831,
      "logps/chosen": -175.20233154296875,
      "logps/rejected": -257.7266540527344,
      "loss": 0.0486,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4860222339630127,
      "rewards/margins": 7.3104448318481445,
      "rewards/rejected": -8.796466827392578,
      "step": 1797
    },
    {
      "epoch": 2.35,
      "learning_rate": 6.008479238454915e-06,
      "logits/chosen": -1.8654284477233887,
      "logits/rejected": -1.930616021156311,
      "logps/chosen": -144.3576202392578,
      "logps/rejected": -239.07516479492188,
      "loss": 0.0224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.354351282119751,
      "rewards/margins": 9.111152648925781,
      "rewards/rejected": -9.46550464630127,
      "step": 1798
    },
    {
      "epoch": 2.35,
      "learning_rate": 5.98519766927732e-06,
      "logits/chosen": -1.6634949445724487,
      "logits/rejected": -1.7268803119659424,
      "logps/chosen": -182.5557861328125,
      "logps/rejected": -249.2535400390625,
      "loss": 0.1411,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.8785061836242676,
      "rewards/margins": 6.085895538330078,
      "rewards/rejected": -8.964402198791504,
      "step": 1799
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.961955158134391e-06,
      "logits/chosen": -1.8729782104492188,
      "logits/rejected": -1.8884284496307373,
      "logps/chosen": -141.24949645996094,
      "logps/rejected": -243.7825927734375,
      "loss": 0.0452,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9438889026641846,
      "rewards/margins": 8.604199409484863,
      "rewards/rejected": -9.548088073730469,
      "step": 1800
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.938751752768226e-06,
      "logits/chosen": -1.9610137939453125,
      "logits/rejected": -1.9373916387557983,
      "logps/chosen": -146.06907653808594,
      "logps/rejected": -229.596923828125,
      "loss": 0.0829,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8297717571258545,
      "rewards/margins": 7.841850280761719,
      "rewards/rejected": -8.671622276306152,
      "step": 1801
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.915587500840625e-06,
      "logits/chosen": -1.8033699989318848,
      "logits/rejected": -1.8183770179748535,
      "logps/chosen": -197.17935180664062,
      "logps/rejected": -285.85528564453125,
      "loss": 0.046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9398608803749084,
      "rewards/margins": 9.379088401794434,
      "rewards/rejected": -10.318948745727539,
      "step": 1802
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.892462449932928e-06,
      "logits/chosen": -1.819123387336731,
      "logits/rejected": -1.9960455894470215,
      "logps/chosen": -168.45912170410156,
      "logps/rejected": -326.82342529296875,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.303305059671402,
      "rewards/margins": 10.337949752807617,
      "rewards/rejected": -10.641254425048828,
      "step": 1803
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.869376647545993e-06,
      "logits/chosen": -1.649829387664795,
      "logits/rejected": -1.6216328144073486,
      "logps/chosen": -176.91773986816406,
      "logps/rejected": -269.35223388671875,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.14230845868587494,
      "rewards/margins": 10.178213119506836,
      "rewards/rejected": -10.32052230834961,
      "step": 1804
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.84633014110002e-06,
      "logits/chosen": -1.8258816003799438,
      "logits/rejected": -1.8760173320770264,
      "logps/chosen": -151.986083984375,
      "logps/rejected": -261.3778076171875,
      "loss": 0.045,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0521061420440674,
      "rewards/margins": 9.480091094970703,
      "rewards/rejected": -10.532196998596191,
      "step": 1805
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.82332297793452e-06,
      "logits/chosen": -1.5174182653427124,
      "logits/rejected": -1.5330994129180908,
      "logps/chosen": -221.64381408691406,
      "logps/rejected": -292.15313720703125,
      "loss": 0.0478,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.920825719833374,
      "rewards/margins": 7.789780616760254,
      "rewards/rejected": -10.710606575012207,
      "step": 1806
    },
    {
      "epoch": 2.36,
      "learning_rate": 5.800355205308183e-06,
      "logits/chosen": -1.8786711692810059,
      "logits/rejected": -1.9063522815704346,
      "logps/chosen": -157.91378784179688,
      "logps/rejected": -255.27713012695312,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.867735743522644,
      "rewards/margins": 10.232573509216309,
      "rewards/rejected": -11.100308418273926,
      "step": 1807
    },
    {
      "epoch": 2.37,
      "learning_rate": 5.777426870398777e-06,
      "logits/chosen": -1.6874775886535645,
      "logits/rejected": -1.733237862586975,
      "logps/chosen": -175.5268096923828,
      "logps/rejected": -249.50299072265625,
      "loss": 0.0733,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.328782558441162,
      "rewards/margins": 7.430053234100342,
      "rewards/rejected": -8.758835792541504,
      "step": 1808
    },
    {
      "epoch": 2.37,
      "learning_rate": 5.754538020303063e-06,
      "logits/chosen": -1.644497275352478,
      "logits/rejected": -1.6909924745559692,
      "logps/chosen": -153.493408203125,
      "logps/rejected": -251.7250213623047,
      "loss": 0.0437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7651001214981079,
      "rewards/margins": 10.252270698547363,
      "rewards/rejected": -11.017372131347656,
      "step": 1809
    },
    {
      "epoch": 2.37,
      "learning_rate": 5.731688702036717e-06,
      "logits/chosen": -1.6168606281280518,
      "logits/rejected": -1.593480110168457,
      "logps/chosen": -168.53077697753906,
      "logps/rejected": -246.25689697265625,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7803144454956055,
      "rewards/margins": 9.200432777404785,
      "rewards/rejected": -9.98074722290039,
      "step": 1810
    },
    {
      "epoch": 2.37,
      "learning_rate": 5.708878962534181e-06,
      "logits/chosen": -1.644266963005066,
      "logits/rejected": -1.725996971130371,
      "logps/chosen": -196.8369903564453,
      "logps/rejected": -284.9306335449219,
      "loss": 0.0887,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2806141376495361,
      "rewards/margins": 7.755922317504883,
      "rewards/rejected": -9.03653621673584,
      "step": 1811
    },
    {
      "epoch": 2.37,
      "learning_rate": 5.686108848648624e-06,
      "logits/chosen": -1.9136842489242554,
      "logits/rejected": -1.8967397212982178,
      "logps/chosen": -173.3729705810547,
      "logps/rejected": -237.43865966796875,
      "loss": 0.0449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2968072891235352,
      "rewards/margins": 8.209379196166992,
      "rewards/rejected": -9.506186485290527,
      "step": 1812
    },
    {
      "epoch": 2.37,
      "learning_rate": 5.6633784071518205e-06,
      "logits/chosen": -1.913116216659546,
      "logits/rejected": -1.9241000413894653,
      "logps/chosen": -202.77630615234375,
      "logps/rejected": -318.6482238769531,
      "loss": 0.0022,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3474459648132324,
      "rewards/margins": 10.188543319702148,
      "rewards/rejected": -11.535989761352539,
      "step": 1813
    },
    {
      "epoch": 2.37,
      "learning_rate": 5.640687684734039e-06,
      "logits/chosen": -1.6127270460128784,
      "logits/rejected": -1.6668362617492676,
      "logps/chosen": -162.12420654296875,
      "logps/rejected": -250.98828125,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2485980987548828,
      "rewards/margins": 8.602983474731445,
      "rewards/rejected": -9.851581573486328,
      "step": 1814
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.618036728003965e-06,
      "logits/chosen": -1.7309653759002686,
      "logits/rejected": -1.8025567531585693,
      "logps/chosen": -155.402099609375,
      "logps/rejected": -249.3150634765625,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3402220010757446,
      "rewards/margins": 8.50476360321045,
      "rewards/rejected": -9.844985961914062,
      "step": 1815
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.595425583488608e-06,
      "logits/chosen": -1.8820011615753174,
      "logits/rejected": -1.9767611026763916,
      "logps/chosen": -207.30712890625,
      "logps/rejected": -298.0267333984375,
      "loss": 0.0914,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.134304404258728,
      "rewards/margins": 7.299224376678467,
      "rewards/rejected": -8.433528900146484,
      "step": 1816
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.572854297633209e-06,
      "logits/chosen": -1.7283885478973389,
      "logits/rejected": -1.7608424425125122,
      "logps/chosen": -154.72589111328125,
      "logps/rejected": -284.1565856933594,
      "loss": 0.0874,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.17979025840759277,
      "rewards/margins": 11.754396438598633,
      "rewards/rejected": -11.934185981750488,
      "step": 1817
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.550322916801115e-06,
      "logits/chosen": -1.791946291923523,
      "logits/rejected": -1.8327200412750244,
      "logps/chosen": -164.39732360839844,
      "logps/rejected": -231.184326171875,
      "loss": 0.0162,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.624418020248413,
      "rewards/margins": 7.650132656097412,
      "rewards/rejected": -9.274550437927246,
      "step": 1818
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.5278314872737105e-06,
      "logits/chosen": -1.4700098037719727,
      "logits/rejected": -1.4394402503967285,
      "logps/chosen": -174.79052734375,
      "logps/rejected": -275.8683776855469,
      "loss": 0.0446,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3129717111587524,
      "rewards/margins": 9.203598976135254,
      "rewards/rejected": -10.516571044921875,
      "step": 1819
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.505380055250325e-06,
      "logits/chosen": -1.9886549711227417,
      "logits/rejected": -1.9537912607192993,
      "logps/chosen": -148.13975524902344,
      "logps/rejected": -238.25772094726562,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.409706711769104,
      "rewards/margins": 7.884661674499512,
      "rewards/rejected": -9.294366836547852,
      "step": 1820
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.482968666848132e-06,
      "logits/chosen": -1.8707071542739868,
      "logits/rejected": -1.9022600650787354,
      "logps/chosen": -150.8856964111328,
      "logps/rejected": -265.9314270019531,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6483213901519775,
      "rewards/margins": 10.523534774780273,
      "rewards/rejected": -11.171855926513672,
      "step": 1821
    },
    {
      "epoch": 2.38,
      "learning_rate": 5.460597368102033e-06,
      "logits/chosen": -1.7832237482070923,
      "logits/rejected": -1.789170742034912,
      "logps/chosen": -167.95005798339844,
      "logps/rejected": -222.56802368164062,
      "loss": 0.0076,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.214505672454834,
      "rewards/margins": 6.913451671600342,
      "rewards/rejected": -8.127957344055176,
      "step": 1822
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.4382662049646036e-06,
      "logits/chosen": -1.7512027025222778,
      "logits/rejected": -1.8256597518920898,
      "logps/chosen": -167.52328491210938,
      "logps/rejected": -267.4353942871094,
      "loss": 0.0456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.32216790318489075,
      "rewards/margins": 9.833447456359863,
      "rewards/rejected": -10.15561580657959,
      "step": 1823
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.4159752233059745e-06,
      "logits/chosen": -1.8599250316619873,
      "logits/rejected": -1.8324565887451172,
      "logps/chosen": -162.7130126953125,
      "logps/rejected": -228.86544799804688,
      "loss": 0.0899,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7308731079101562,
      "rewards/margins": 7.319431304931641,
      "rewards/rejected": -9.050304412841797,
      "step": 1824
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.393724468913713e-06,
      "logits/chosen": -1.685518503189087,
      "logits/rejected": -1.647742509841919,
      "logps/chosen": -181.34681701660156,
      "logps/rejected": -260.39471435546875,
      "loss": 0.1056,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.425970196723938,
      "rewards/margins": 7.681146144866943,
      "rewards/rejected": -9.107115745544434,
      "step": 1825
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.371513987492788e-06,
      "logits/chosen": -1.4467105865478516,
      "logits/rejected": -1.4234956502914429,
      "logps/chosen": -176.7122039794922,
      "logps/rejected": -301.3267822265625,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.8961801528930664,
      "rewards/margins": 9.903864860534668,
      "rewards/rejected": -11.800045013427734,
      "step": 1826
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.34934382466544e-06,
      "logits/chosen": -1.7407598495483398,
      "logits/rejected": -1.6466485261917114,
      "logps/chosen": -177.20155334472656,
      "logps/rejected": -270.992431640625,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.22611603140830994,
      "rewards/margins": 9.306327819824219,
      "rewards/rejected": -9.532442092895508,
      "step": 1827
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.32721402597107e-06,
      "logits/chosen": -1.8147292137145996,
      "logits/rejected": -1.8035141229629517,
      "logps/chosen": -137.78890991210938,
      "logps/rejected": -220.59375,
      "loss": 0.0476,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1758556365966797,
      "rewards/margins": 7.639559745788574,
      "rewards/rejected": -8.815414428710938,
      "step": 1828
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.3051246368661965e-06,
      "logits/chosen": -1.866050124168396,
      "logits/rejected": -1.841845989227295,
      "logps/chosen": -158.92559814453125,
      "logps/rejected": -240.94903564453125,
      "loss": 0.0511,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7676749229431152,
      "rewards/margins": 7.427980422973633,
      "rewards/rejected": -9.195655822753906,
      "step": 1829
    },
    {
      "epoch": 2.39,
      "learning_rate": 5.283075702724305e-06,
      "logits/chosen": -1.8375452756881714,
      "logits/rejected": -1.7923805713653564,
      "logps/chosen": -177.02615356445312,
      "logps/rejected": -259.9975280761719,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.064595937728882,
      "rewards/margins": 8.66468620300293,
      "rewards/rejected": -10.729281425476074,
      "step": 1830
    },
    {
      "epoch": 2.4,
      "learning_rate": 5.261067268835812e-06,
      "logits/chosen": -1.8552744388580322,
      "logits/rejected": -1.8669440746307373,
      "logps/chosen": -181.46368408203125,
      "logps/rejected": -282.1120300292969,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.532908320426941,
      "rewards/margins": 9.032302856445312,
      "rewards/rejected": -10.56521224975586,
      "step": 1831
    },
    {
      "epoch": 2.4,
      "learning_rate": 5.239099380407916e-06,
      "logits/chosen": -1.9588475227355957,
      "logits/rejected": -1.9900915622711182,
      "logps/chosen": -228.31106567382812,
      "logps/rejected": -360.1578369140625,
      "loss": 0.0872,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2150044441223145,
      "rewards/margins": 10.900214195251465,
      "rewards/rejected": -12.115219116210938,
      "step": 1832
    },
    {
      "epoch": 2.4,
      "learning_rate": 5.217172082564547e-06,
      "logits/chosen": -1.9226619005203247,
      "logits/rejected": -1.8748016357421875,
      "logps/chosen": -164.7508087158203,
      "logps/rejected": -281.2464599609375,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5262323617935181,
      "rewards/margins": 10.75429916381836,
      "rewards/rejected": -11.280531883239746,
      "step": 1833
    },
    {
      "epoch": 2.4,
      "learning_rate": 5.195285420346263e-06,
      "logits/chosen": -1.893693208694458,
      "logits/rejected": -1.845407485961914,
      "logps/chosen": -149.87037658691406,
      "logps/rejected": -193.39797973632812,
      "loss": 0.135,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.5354095697402954,
      "rewards/margins": 5.375506401062012,
      "rewards/rejected": -6.910915851593018,
      "step": 1834
    },
    {
      "epoch": 2.4,
      "learning_rate": 5.17343943871014e-06,
      "logits/chosen": -1.921462059020996,
      "logits/rejected": -1.9630136489868164,
      "logps/chosen": -186.42306518554688,
      "logps/rejected": -272.7850036621094,
      "loss": 0.0881,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0841878652572632,
      "rewards/margins": 7.272481441497803,
      "rewards/rejected": -8.356668472290039,
      "step": 1835
    },
    {
      "epoch": 2.4,
      "learning_rate": 5.151634182529691e-06,
      "logits/chosen": -1.8229271173477173,
      "logits/rejected": -1.8508105278015137,
      "logps/chosen": -159.3003692626953,
      "logps/rejected": -243.13243103027344,
      "loss": 0.0037,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5930259227752686,
      "rewards/margins": 7.876507759094238,
      "rewards/rejected": -9.469533920288086,
      "step": 1836
    },
    {
      "epoch": 2.4,
      "learning_rate": 5.129869696594786e-06,
      "logits/chosen": -1.6906994581222534,
      "logits/rejected": -1.6776598691940308,
      "logps/chosen": -153.01341247558594,
      "logps/rejected": -260.2983703613281,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0089322328567505,
      "rewards/margins": 9.491535186767578,
      "rewards/rejected": -10.500467300415039,
      "step": 1837
    },
    {
      "epoch": 2.41,
      "learning_rate": 5.108146025611554e-06,
      "logits/chosen": -2.090075731277466,
      "logits/rejected": -2.1263160705566406,
      "logps/chosen": -157.9957733154297,
      "logps/rejected": -219.81924438476562,
      "loss": 0.0887,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.494313359260559,
      "rewards/margins": 6.957061767578125,
      "rewards/rejected": -8.451374053955078,
      "step": 1838
    },
    {
      "epoch": 2.41,
      "learning_rate": 5.086463214202264e-06,
      "logits/chosen": -1.7797666788101196,
      "logits/rejected": -1.826112985610962,
      "logps/chosen": -149.82540893554688,
      "logps/rejected": -257.0027160644531,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.06566246598958969,
      "rewards/margins": 10.07011604309082,
      "rewards/rejected": -10.135777473449707,
      "step": 1839
    },
    {
      "epoch": 2.41,
      "learning_rate": 5.064821306905288e-06,
      "logits/chosen": -1.8234814405441284,
      "logits/rejected": -1.809545636177063,
      "logps/chosen": -173.8544921875,
      "logps/rejected": -281.5364074707031,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.4215404987335205,
      "rewards/margins": 8.993759155273438,
      "rewards/rejected": -11.415298461914062,
      "step": 1840
    },
    {
      "epoch": 2.41,
      "learning_rate": 5.043220348174945e-06,
      "logits/chosen": -1.9463227987289429,
      "logits/rejected": -2.0307672023773193,
      "logps/chosen": -157.9526824951172,
      "logps/rejected": -258.4635925292969,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8971003890037537,
      "rewards/margins": 9.50979995727539,
      "rewards/rejected": -10.406900405883789,
      "step": 1841
    },
    {
      "epoch": 2.41,
      "learning_rate": 5.021660382381457e-06,
      "logits/chosen": -1.921215295791626,
      "logits/rejected": -2.0194950103759766,
      "logps/chosen": -152.3760528564453,
      "logps/rejected": -257.3869934082031,
      "loss": 0.044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6928465366363525,
      "rewards/margins": 9.074962615966797,
      "rewards/rejected": -9.767809867858887,
      "step": 1842
    },
    {
      "epoch": 2.41,
      "learning_rate": 5.000141453810847e-06,
      "logits/chosen": -1.856144666671753,
      "logits/rejected": -1.851601481437683,
      "logps/chosen": -182.4647216796875,
      "logps/rejected": -278.25494384765625,
      "loss": 0.0879,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.115578532218933,
      "rewards/margins": 8.696277618408203,
      "rewards/rejected": -9.811856269836426,
      "step": 1843
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.9786636066648436e-06,
      "logits/chosen": -1.9953882694244385,
      "logits/rejected": -1.9421712160110474,
      "logps/chosen": -189.7842559814453,
      "logps/rejected": -287.136962890625,
      "loss": 0.0012,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.051191806793213,
      "rewards/margins": 9.621888160705566,
      "rewards/rejected": -10.673080444335938,
      "step": 1844
    },
    {
      "epoch": 2.41,
      "learning_rate": 4.957226885060779e-06,
      "logits/chosen": -1.5074043273925781,
      "logits/rejected": -1.5145354270935059,
      "logps/chosen": -158.01559448242188,
      "logps/rejected": -266.94635009765625,
      "loss": 0.0875,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3148698806762695,
      "rewards/margins": 8.630085945129395,
      "rewards/rejected": -9.944955825805664,
      "step": 1845
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.935831333031527e-06,
      "logits/chosen": -1.982537031173706,
      "logits/rejected": -1.9602676630020142,
      "logps/chosen": -169.39596557617188,
      "logps/rejected": -215.37237548828125,
      "loss": 0.0918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1537246704101562,
      "rewards/margins": 6.32325553894043,
      "rewards/rejected": -8.476980209350586,
      "step": 1846
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.914476994525372e-06,
      "logits/chosen": -1.686547040939331,
      "logits/rejected": -1.7233699560165405,
      "logps/chosen": -160.05368041992188,
      "logps/rejected": -232.4645538330078,
      "loss": 0.0879,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1599763631820679,
      "rewards/margins": 6.913029670715332,
      "rewards/rejected": -8.073005676269531,
      "step": 1847
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.893163913405971e-06,
      "logits/chosen": -1.8559006452560425,
      "logits/rejected": -1.8504712581634521,
      "logps/chosen": -248.82484436035156,
      "logps/rejected": -323.56439208984375,
      "loss": 0.0891,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6479015350341797,
      "rewards/margins": 7.927282333374023,
      "rewards/rejected": -8.575182914733887,
      "step": 1848
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.871892133452211e-06,
      "logits/chosen": -1.8911268711090088,
      "logits/rejected": -1.8310123682022095,
      "logps/chosen": -181.1059112548828,
      "logps/rejected": -273.0307312011719,
      "loss": 0.0218,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2174774408340454,
      "rewards/margins": 10.580732345581055,
      "rewards/rejected": -11.798210144042969,
      "step": 1849
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.850661698358156e-06,
      "logits/chosen": -1.7013300657272339,
      "logits/rejected": -1.7378973960876465,
      "logps/chosen": -176.54147338867188,
      "logps/rejected": -298.0980529785156,
      "loss": 0.0521,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8014607429504395,
      "rewards/margins": 10.176857948303223,
      "rewards/rejected": -11.978318214416504,
      "step": 1850
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.8294726517329496e-06,
      "logits/chosen": -1.7144262790679932,
      "logits/rejected": -1.7501251697540283,
      "logps/chosen": -156.00686645507812,
      "logps/rejected": -239.52749633789062,
      "loss": 0.0449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1516990661621094,
      "rewards/margins": 8.29002857208252,
      "rewards/rejected": -9.441727638244629,
      "step": 1851
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.808325037100691e-06,
      "logits/chosen": -1.9031648635864258,
      "logits/rejected": -1.9143873453140259,
      "logps/chosen": -184.77984619140625,
      "logps/rejected": -286.36920166015625,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4094650745391846,
      "rewards/margins": 9.453594207763672,
      "rewards/rejected": -10.863058090209961,
      "step": 1852
    },
    {
      "epoch": 2.42,
      "learning_rate": 4.787218897900403e-06,
      "logits/chosen": -1.9727962017059326,
      "logits/rejected": -1.9811248779296875,
      "logps/chosen": -213.87249755859375,
      "logps/rejected": -283.25390625,
      "loss": 0.2188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.2139840126037598,
      "rewards/margins": 6.789841651916504,
      "rewards/rejected": -9.003825187683105,
      "step": 1853
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.766154277485915e-06,
      "logits/chosen": -1.9707348346710205,
      "logits/rejected": -2.061739206314087,
      "logps/chosen": -170.50668334960938,
      "logps/rejected": -256.5799560546875,
      "loss": 0.0444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6609563827514648,
      "rewards/margins": 9.082340240478516,
      "rewards/rejected": -9.743297576904297,
      "step": 1854
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.745131219125748e-06,
      "logits/chosen": -1.7594854831695557,
      "logits/rejected": -1.8002958297729492,
      "logps/chosen": -208.25477600097656,
      "logps/rejected": -293.4322814941406,
      "loss": 0.0451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.366346001625061,
      "rewards/margins": 7.665379524230957,
      "rewards/rejected": -9.03172492980957,
      "step": 1855
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.7241497660030744e-06,
      "logits/chosen": -1.832306146621704,
      "logits/rejected": -1.8248895406723022,
      "logps/chosen": -162.09207153320312,
      "logps/rejected": -288.51739501953125,
      "loss": 0.0026,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.546169638633728,
      "rewards/margins": 10.596427917480469,
      "rewards/rejected": -12.142598152160645,
      "step": 1856
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.703209961215607e-06,
      "logits/chosen": -1.925199031829834,
      "logits/rejected": -1.9174048900604248,
      "logps/chosen": -168.96656799316406,
      "logps/rejected": -250.91664123535156,
      "loss": 0.0073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7092611193656921,
      "rewards/margins": 8.274471282958984,
      "rewards/rejected": -8.983732223510742,
      "step": 1857
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.682311847775489e-06,
      "logits/chosen": -1.7236016988754272,
      "logits/rejected": -1.788609266281128,
      "logps/chosen": -169.9625244140625,
      "logps/rejected": -257.8612976074219,
      "loss": 0.0907,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5359935760498047,
      "rewards/margins": 8.339431762695312,
      "rewards/rejected": -9.875425338745117,
      "step": 1858
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.661455468609235e-06,
      "logits/chosen": -1.427506685256958,
      "logits/rejected": -1.455967664718628,
      "logps/chosen": -171.5054931640625,
      "logps/rejected": -220.9553985595703,
      "loss": 0.1325,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.2707475423812866,
      "rewards/margins": 6.626494884490967,
      "rewards/rejected": -7.897242546081543,
      "step": 1859
    },
    {
      "epoch": 2.43,
      "learning_rate": 4.640640866557644e-06,
      "logits/chosen": -1.8355470895767212,
      "logits/rejected": -1.9365921020507812,
      "logps/chosen": -160.73928833007812,
      "logps/rejected": -266.6350402832031,
      "loss": 0.0443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4644968509674072,
      "rewards/margins": 8.865848541259766,
      "rewards/rejected": -10.330345153808594,
      "step": 1860
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.6198680843756975e-06,
      "logits/chosen": -1.7328015565872192,
      "logits/rejected": -1.7365508079528809,
      "logps/chosen": -224.6548309326172,
      "logps/rejected": -314.62493896484375,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1965479850769043,
      "rewards/margins": 8.27452278137207,
      "rewards/rejected": -9.471071243286133,
      "step": 1861
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.599137164732464e-06,
      "logits/chosen": -1.7878057956695557,
      "logits/rejected": -1.6829179525375366,
      "logps/chosen": -222.8665008544922,
      "logps/rejected": -332.0810241699219,
      "loss": 0.0452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.8386578559875488,
      "rewards/margins": 11.640946388244629,
      "rewards/rejected": -12.47960376739502,
      "step": 1862
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.578448150211026e-06,
      "logits/chosen": -1.6933939456939697,
      "logits/rejected": -1.72417414188385,
      "logps/chosen": -150.51979064941406,
      "logps/rejected": -237.14224243164062,
      "loss": 0.0035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3049812316894531,
      "rewards/margins": 8.090076446533203,
      "rewards/rejected": -9.395058631896973,
      "step": 1863
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.557801083308403e-06,
      "logits/chosen": -1.7908655405044556,
      "logits/rejected": -1.8133585453033447,
      "logps/chosen": -159.14002990722656,
      "logps/rejected": -237.7046356201172,
      "loss": 0.1311,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.6292684078216553,
      "rewards/margins": 7.855827331542969,
      "rewards/rejected": -9.485095977783203,
      "step": 1864
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.53719600643544e-06,
      "logits/chosen": -1.893837332725525,
      "logits/rejected": -1.9021137952804565,
      "logps/chosen": -142.3712921142578,
      "logps/rejected": -198.23330688476562,
      "loss": 0.0459,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5361227989196777,
      "rewards/margins": 6.980817794799805,
      "rewards/rejected": -7.516941070556641,
      "step": 1865
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.516632961916722e-06,
      "logits/chosen": -1.9697438478469849,
      "logits/rejected": -2.0294458866119385,
      "logps/chosen": -181.87208557128906,
      "logps/rejected": -285.9231872558594,
      "loss": 0.0469,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7394585013389587,
      "rewards/margins": 9.006132125854492,
      "rewards/rejected": -9.745590209960938,
      "step": 1866
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.496111991990518e-06,
      "logits/chosen": -2.0269381999969482,
      "logits/rejected": -1.9887118339538574,
      "logps/chosen": -175.04978942871094,
      "logps/rejected": -265.1463623046875,
      "loss": 0.0056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.5267537236213684,
      "rewards/margins": 10.348395347595215,
      "rewards/rejected": -10.87514877319336,
      "step": 1867
    },
    {
      "epoch": 2.44,
      "learning_rate": 4.475633138808663e-06,
      "logits/chosen": -2.0939102172851562,
      "logits/rejected": -2.007652997970581,
      "logps/chosen": -139.40298461914062,
      "logps/rejected": -225.70179748535156,
      "loss": 0.0877,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6997631788253784,
      "rewards/margins": 8.688980102539062,
      "rewards/rejected": -9.388744354248047,
      "step": 1868
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.45519644443646e-06,
      "logits/chosen": -1.9846158027648926,
      "logits/rejected": -1.9977834224700928,
      "logps/chosen": -162.92637634277344,
      "logps/rejected": -258.0967712402344,
      "loss": 0.047,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.228029489517212,
      "rewards/margins": 8.419919967651367,
      "rewards/rejected": -9.647950172424316,
      "step": 1869
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.434801950852644e-06,
      "logits/chosen": -1.6758570671081543,
      "logits/rejected": -1.6716039180755615,
      "logps/chosen": -164.8556365966797,
      "logps/rejected": -240.500732421875,
      "loss": 0.0625,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3664546012878418,
      "rewards/margins": 8.135287284851074,
      "rewards/rejected": -9.501742362976074,
      "step": 1870
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.414449699949255e-06,
      "logits/chosen": -1.897088885307312,
      "logits/rejected": -1.9522241353988647,
      "logps/chosen": -206.5328826904297,
      "logps/rejected": -306.1324157714844,
      "loss": 0.0874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0038398504257202,
      "rewards/margins": 8.901524543762207,
      "rewards/rejected": -9.905364036560059,
      "step": 1871
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.394139733531555e-06,
      "logits/chosen": -1.727203130722046,
      "logits/rejected": -1.6950209140777588,
      "logps/chosen": -204.94601440429688,
      "logps/rejected": -236.50953674316406,
      "loss": 0.177,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.6954196095466614,
      "rewards/margins": 6.443286418914795,
      "rewards/rejected": -7.138706207275391,
      "step": 1872
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.373872093317965e-06,
      "logits/chosen": -2.11216402053833,
      "logits/rejected": -2.1386733055114746,
      "logps/chosen": -159.49911499023438,
      "logps/rejected": -240.71621704101562,
      "loss": 0.1303,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4864517450332642,
      "rewards/margins": 7.543427467346191,
      "rewards/rejected": -9.029878616333008,
      "step": 1873
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.353646820939944e-06,
      "logits/chosen": -1.8664822578430176,
      "logits/rejected": -1.8709962368011475,
      "logps/chosen": -150.19898986816406,
      "logps/rejected": -252.53526306152344,
      "loss": 0.0443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5000903606414795,
      "rewards/margins": 8.614351272583008,
      "rewards/rejected": -10.114441871643066,
      "step": 1874
    },
    {
      "epoch": 2.45,
      "learning_rate": 4.333463957941952e-06,
      "logits/chosen": -1.9424302577972412,
      "logits/rejected": -1.9463444948196411,
      "logps/chosen": -174.458740234375,
      "logps/rejected": -277.8190002441406,
      "loss": 0.0459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.699613332748413,
      "rewards/margins": 9.456645965576172,
      "rewards/rejected": -11.15626049041748,
      "step": 1875
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.313323545781306e-06,
      "logits/chosen": -1.8622289896011353,
      "logits/rejected": -1.7156347036361694,
      "logps/chosen": -203.263427734375,
      "logps/rejected": -266.94256591796875,
      "loss": 0.0016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.2786290645599365,
      "rewards/margins": 8.69839859008789,
      "rewards/rejected": -10.977028846740723,
      "step": 1876
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.293225625828143e-06,
      "logits/chosen": -1.6078665256500244,
      "logits/rejected": -1.6221495866775513,
      "logps/chosen": -157.21278381347656,
      "logps/rejected": -245.13694763183594,
      "loss": 0.0015,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1403205394744873,
      "rewards/margins": 8.569680213928223,
      "rewards/rejected": -9.710000991821289,
      "step": 1877
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.273170239365323e-06,
      "logits/chosen": -2.04641056060791,
      "logits/rejected": -2.0428082942962646,
      "logps/chosen": -218.95347595214844,
      "logps/rejected": -330.5970764160156,
      "loss": 0.0442,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2602277994155884,
      "rewards/margins": 10.10246467590332,
      "rewards/rejected": -11.362691879272461,
      "step": 1878
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.253157427588325e-06,
      "logits/chosen": -1.9846681356430054,
      "logits/rejected": -1.9943360090255737,
      "logps/chosen": -200.45144653320312,
      "logps/rejected": -291.7164611816406,
      "loss": 0.1758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.3286120891571045,
      "rewards/margins": 6.549439907073975,
      "rewards/rejected": -8.8780517578125,
      "step": 1879
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.233187231605173e-06,
      "logits/chosen": -1.846575140953064,
      "logits/rejected": -1.8467507362365723,
      "logps/chosen": -203.3041534423828,
      "logps/rejected": -289.2364501953125,
      "loss": 0.1315,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.1249494552612305,
      "rewards/margins": 7.693889141082764,
      "rewards/rejected": -8.818838119506836,
      "step": 1880
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.213259692436367e-06,
      "logits/chosen": -1.998410940170288,
      "logits/rejected": -1.984098196029663,
      "logps/chosen": -194.3431396484375,
      "logps/rejected": -277.8040466308594,
      "loss": 0.0039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.6438612937927246,
      "rewards/margins": 8.47049617767334,
      "rewards/rejected": -11.114357948303223,
      "step": 1881
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.193374851014789e-06,
      "logits/chosen": -2.180818796157837,
      "logits/rejected": -2.1985692977905273,
      "logps/chosen": -142.76791381835938,
      "logps/rejected": -250.35308837890625,
      "loss": 0.089,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.5618703365325928,
      "rewards/margins": 9.195538520812988,
      "rewards/rejected": -10.757408142089844,
      "step": 1882
    },
    {
      "epoch": 2.46,
      "learning_rate": 4.1735327481855965e-06,
      "logits/chosen": -1.9301328659057617,
      "logits/rejected": -1.9123741388320923,
      "logps/chosen": -171.02801513671875,
      "logps/rejected": -249.6219024658203,
      "loss": 0.0452,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0056562423706055,
      "rewards/margins": 7.982565879821777,
      "rewards/rejected": -8.988222122192383,
      "step": 1883
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.153733424706183e-06,
      "logits/chosen": -2.0726072788238525,
      "logits/rejected": -2.102792978286743,
      "logps/chosen": -205.4851837158203,
      "logps/rejected": -296.5257568359375,
      "loss": 0.0458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0603342056274414,
      "rewards/margins": 8.587060928344727,
      "rewards/rejected": -10.647395133972168,
      "step": 1884
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.13397692124605e-06,
      "logits/chosen": -1.7905073165893555,
      "logits/rejected": -1.8003227710723877,
      "logps/chosen": -204.98776245117188,
      "logps/rejected": -275.7078552246094,
      "loss": 0.0526,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8408176898956299,
      "rewards/margins": 6.810851097106934,
      "rewards/rejected": -8.651668548583984,
      "step": 1885
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.114263278386743e-06,
      "logits/chosen": -1.870998740196228,
      "logits/rejected": -1.8375709056854248,
      "logps/chosen": -208.5353240966797,
      "logps/rejected": -310.5347900390625,
      "loss": 0.0497,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1676502227783203,
      "rewards/margins": 8.382265090942383,
      "rewards/rejected": -9.549915313720703,
      "step": 1886
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.09459253662178e-06,
      "logits/chosen": -1.7835994958877563,
      "logits/rejected": -1.833801507949829,
      "logps/chosen": -163.20249938964844,
      "logps/rejected": -235.98446655273438,
      "loss": 0.0464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2792348861694336,
      "rewards/margins": 7.6134724617004395,
      "rewards/rejected": -8.892707824707031,
      "step": 1887
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.074964736356563e-06,
      "logits/chosen": -1.9710181951522827,
      "logits/rejected": -1.9776310920715332,
      "logps/chosen": -166.91244506835938,
      "logps/rejected": -260.22833251953125,
      "loss": 0.0445,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.635888397693634,
      "rewards/margins": 9.668231010437012,
      "rewards/rejected": -10.304119110107422,
      "step": 1888
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.055379917908258e-06,
      "logits/chosen": -1.9696500301361084,
      "logits/rejected": -1.9052053689956665,
      "logps/chosen": -148.14048767089844,
      "logps/rejected": -222.27215576171875,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2268353700637817,
      "rewards/margins": 8.127723693847656,
      "rewards/rejected": -9.354558944702148,
      "step": 1889
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.035838121505778e-06,
      "logits/chosen": -1.9344145059585571,
      "logits/rejected": -1.9301822185516357,
      "logps/chosen": -194.29672241210938,
      "logps/rejected": -295.50714111328125,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2769558429718018,
      "rewards/margins": 10.218704223632812,
      "rewards/rejected": -11.495660781860352,
      "step": 1890
    },
    {
      "epoch": 2.47,
      "learning_rate": 4.016339387289636e-06,
      "logits/chosen": -1.9737623929977417,
      "logits/rejected": -2.016094923019409,
      "logps/chosen": -192.3032684326172,
      "logps/rejected": -288.0450134277344,
      "loss": 0.0456,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8436172008514404,
      "rewards/margins": 8.860701560974121,
      "rewards/rejected": -10.704319953918457,
      "step": 1891
    },
    {
      "epoch": 2.48,
      "learning_rate": 3.996883755311917e-06,
      "logits/chosen": -1.7308728694915771,
      "logits/rejected": -1.7060117721557617,
      "logps/chosen": -185.12342834472656,
      "logps/rejected": -263.79302978515625,
      "loss": 0.131,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -2.0738894939422607,
      "rewards/margins": 7.3270158767700195,
      "rewards/rejected": -9.400903701782227,
      "step": 1892
    },
    {
      "epoch": 2.48,
      "learning_rate": 3.977471265536142e-06,
      "logits/chosen": -2.000117778778076,
      "logits/rejected": -2.0036187171936035,
      "logps/chosen": -167.9659423828125,
      "logps/rejected": -248.19422912597656,
      "loss": 0.0017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4795580208301544,
      "rewards/margins": 8.6856107711792,
      "rewards/rejected": -9.165167808532715,
      "step": 1893
    },
    {
      "epoch": 2.48,
      "learning_rate": 3.95810195783724e-06,
      "logits/chosen": -1.9586573839187622,
      "logits/rejected": -2.0055723190307617,
      "logps/chosen": -215.33938598632812,
      "logps/rejected": -326.2347412109375,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.430414080619812,
      "rewards/margins": 8.864473342895508,
      "rewards/rejected": -10.29488754272461,
      "step": 1894
    },
    {
      "epoch": 2.48,
      "learning_rate": 3.938775872001441e-06,
      "logits/chosen": -2.015662908554077,
      "logits/rejected": -2.0366337299346924,
      "logps/chosen": -183.64306640625,
      "logps/rejected": -259.4669189453125,
      "loss": 0.0467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.1929130554199219,
      "rewards/margins": 8.386341094970703,
      "rewards/rejected": -9.579254150390625,
      "step": 1895
    },
    {
      "epoch": 2.48,
      "learning_rate": 3.919493047726156e-06,
      "logits/chosen": -1.9060460329055786,
      "logits/rejected": -1.963003158569336,
      "logps/chosen": -160.76104736328125,
      "logps/rejected": -230.80926513671875,
      "loss": 0.0524,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2992582321166992,
      "rewards/margins": 7.546978950500488,
      "rewards/rejected": -8.846236228942871,
      "step": 1896
    },
    {
      "epoch": 2.48,
      "learning_rate": 3.900253524619973e-06,
      "logits/chosen": -1.9995012283325195,
      "logits/rejected": -1.9741090536117554,
      "logps/chosen": -180.080078125,
      "logps/rejected": -254.02597045898438,
      "loss": 0.1442,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8740156888961792,
      "rewards/margins": 7.707704544067383,
      "rewards/rejected": -9.581720352172852,
      "step": 1897
    },
    {
      "epoch": 2.48,
      "learning_rate": 3.881057342202532e-06,
      "logits/chosen": -1.8846170902252197,
      "logits/rejected": -1.9213114976882935,
      "logps/chosen": -162.14669799804688,
      "logps/rejected": -243.04693603515625,
      "loss": 0.0441,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7811854481697083,
      "rewards/margins": 8.185133934020996,
      "rewards/rejected": -8.966320037841797,
      "step": 1898
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.861904539904421e-06,
      "logits/chosen": -1.7574738264083862,
      "logits/rejected": -1.8166627883911133,
      "logps/chosen": -160.38192749023438,
      "logps/rejected": -253.9595489501953,
      "loss": 0.0058,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0802797079086304,
      "rewards/margins": 9.789755821228027,
      "rewards/rejected": -10.870036125183105,
      "step": 1899
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.842795157067147e-06,
      "logits/chosen": -1.646816611289978,
      "logits/rejected": -1.651963710784912,
      "logps/chosen": -144.14419555664062,
      "logps/rejected": -218.88169860839844,
      "loss": 0.0487,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2004311084747314,
      "rewards/margins": 6.937167644500732,
      "rewards/rejected": -8.137598037719727,
      "step": 1900
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.823729232943027e-06,
      "logits/chosen": -1.9251708984375,
      "logits/rejected": -1.9517838954925537,
      "logps/chosen": -162.0962677001953,
      "logps/rejected": -259.787109375,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.494233250617981,
      "rewards/margins": 8.880905151367188,
      "rewards/rejected": -10.375138282775879,
      "step": 1901
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.804706806695099e-06,
      "logits/chosen": -1.7391762733459473,
      "logits/rejected": -1.706825613975525,
      "logps/chosen": -165.25645446777344,
      "logps/rejected": -251.4967041015625,
      "loss": 0.0029,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4405020475387573,
      "rewards/margins": 8.950372695922852,
      "rewards/rejected": -10.390874862670898,
      "step": 1902
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.785727917397047e-06,
      "logits/chosen": -1.59613835811615,
      "logits/rejected": -1.5825445652008057,
      "logps/chosen": -172.545166015625,
      "logps/rejected": -252.3814239501953,
      "loss": 0.0046,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0128687620162964,
      "rewards/margins": 8.869810104370117,
      "rewards/rejected": -9.882678985595703,
      "step": 1903
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.7667926040331507e-06,
      "logits/chosen": -1.8665236234664917,
      "logits/rejected": -1.9000322818756104,
      "logps/chosen": -170.70303344726562,
      "logps/rejected": -265.84613037109375,
      "loss": 0.0028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0447026491165161,
      "rewards/margins": 9.79289436340332,
      "rewards/rejected": -10.83759593963623,
      "step": 1904
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.7479009054981667e-06,
      "logits/chosen": -1.960362195968628,
      "logits/rejected": -1.9631296396255493,
      "logps/chosen": -171.04489135742188,
      "logps/rejected": -261.0845947265625,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2849982976913452,
      "rewards/margins": 8.437207221984863,
      "rewards/rejected": -9.722206115722656,
      "step": 1905
    },
    {
      "epoch": 2.49,
      "learning_rate": 3.7290528605972625e-06,
      "logits/chosen": -1.9927209615707397,
      "logits/rejected": -1.8980571031570435,
      "logps/chosen": -180.66966247558594,
      "logps/rejected": -254.4411163330078,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9926433563232422,
      "rewards/margins": 8.382731437683105,
      "rewards/rejected": -9.375374794006348,
      "step": 1906
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.7102485080459328e-06,
      "logits/chosen": -1.7959316968917847,
      "logits/rejected": -1.7989314794540405,
      "logps/chosen": -161.15716552734375,
      "logps/rejected": -277.8048400878906,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6480364799499512,
      "rewards/margins": 10.657020568847656,
      "rewards/rejected": -12.305055618286133,
      "step": 1907
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.6914878864699326e-06,
      "logits/chosen": -1.8401857614517212,
      "logits/rejected": -1.8490885496139526,
      "logps/chosen": -180.3686065673828,
      "logps/rejected": -246.40650939941406,
      "loss": 0.0911,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.469294309616089,
      "rewards/margins": 7.560528755187988,
      "rewards/rejected": -10.029823303222656,
      "step": 1908
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.672771034405195e-06,
      "logits/chosen": -1.8544249534606934,
      "logits/rejected": -1.8714629411697388,
      "logps/chosen": -202.76480102539062,
      "logps/rejected": -268.5224609375,
      "loss": 0.0881,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.032835006713867,
      "rewards/margins": 7.587817192077637,
      "rewards/rejected": -9.620651245117188,
      "step": 1909
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.654097990297731e-06,
      "logits/chosen": -1.9069571495056152,
      "logits/rejected": -1.9262149333953857,
      "logps/chosen": -182.80429077148438,
      "logps/rejected": -271.6493835449219,
      "loss": 0.2195,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8242045640945435,
      "rewards/margins": 6.442446708679199,
      "rewards/rejected": -8.266651153564453,
      "step": 1910
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.6354687925035743e-06,
      "logits/chosen": -2.04646372795105,
      "logits/rejected": -2.0554401874542236,
      "logps/chosen": -167.07017517089844,
      "logps/rejected": -303.14862060546875,
      "loss": 0.0436,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0770217180252075,
      "rewards/margins": 11.342137336730957,
      "rewards/rejected": -12.419159889221191,
      "step": 1911
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.6168834792887103e-06,
      "logits/chosen": -1.7153308391571045,
      "logits/rejected": -1.6912052631378174,
      "logps/chosen": -205.31700134277344,
      "logps/rejected": -295.68212890625,
      "loss": 0.0449,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9752507209777832,
      "rewards/margins": 8.712749481201172,
      "rewards/rejected": -9.688000679016113,
      "step": 1912
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.598342088828943e-06,
      "logits/chosen": -2.0192582607269287,
      "logits/rejected": -2.0107357501983643,
      "logps/chosen": -172.4591522216797,
      "logps/rejected": -279.0300598144531,
      "loss": 0.0004,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1163748502731323,
      "rewards/margins": 10.15805435180664,
      "rewards/rejected": -11.274428367614746,
      "step": 1913
    },
    {
      "epoch": 2.5,
      "learning_rate": 3.5798446592098883e-06,
      "logits/chosen": -1.9870482683181763,
      "logits/rejected": -1.9747042655944824,
      "logps/chosen": -157.76092529296875,
      "logps/rejected": -251.87925720214844,
      "loss": 0.0451,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.4392926692962646,
      "rewards/margins": 8.891763687133789,
      "rewards/rejected": -10.331056594848633,
      "step": 1914
    },
    {
      "epoch": 2.51,
      "learning_rate": 3.561391228426861e-06,
      "logits/chosen": -1.854392170906067,
      "logits/rejected": -1.8227899074554443,
      "logps/chosen": -174.9380645751953,
      "logps/rejected": -262.6255187988281,
      "loss": 0.0013,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7663756608963013,
      "rewards/margins": 9.887404441833496,
      "rewards/rejected": -10.653779983520508,
      "step": 1915
    },
    {
      "epoch": 2.51,
      "learning_rate": 3.542981834384776e-06,
      "logits/chosen": -2.070033550262451,
      "logits/rejected": -2.0547399520874023,
      "logps/chosen": -176.39976501464844,
      "logps/rejected": -279.86724853515625,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.25813838839530945,
      "rewards/margins": 11.135327339172363,
      "rewards/rejected": -10.877189636230469,
      "step": 1916
    },
    {
      "epoch": 2.51,
      "learning_rate": 3.5246165148981214e-06,
      "logits/chosen": -1.8121495246887207,
      "logits/rejected": -1.8805229663848877,
      "logps/chosen": -174.73614501953125,
      "logps/rejected": -273.25274658203125,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7994124889373779,
      "rewards/margins": 9.263690948486328,
      "rewards/rejected": -10.063104629516602,
      "step": 1917
    },
    {
      "epoch": 2.51,
      "learning_rate": 3.5062953076908268e-06,
      "logits/chosen": -1.8366397619247437,
      "logits/rejected": -1.7929936647415161,
      "logps/chosen": -169.88864135742188,
      "logps/rejected": -239.2517547607422,
      "loss": 0.0473,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.8681163787841797,
      "rewards/margins": 7.348606109619141,
      "rewards/rejected": -9.21672248840332,
      "step": 1918
    },
    {
      "epoch": 2.51,
      "learning_rate": 3.488018250396233e-06,
      "logits/chosen": -1.7309006452560425,
      "logits/rejected": -1.757127046585083,
      "logps/chosen": -185.51361083984375,
      "logps/rejected": -281.1846008300781,
      "loss": 0.0873,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4568510055541992,
      "rewards/margins": 8.917549133300781,
      "rewards/rejected": -10.374401092529297,
      "step": 1919
    },
    {
      "epoch": 2.51,
      "learning_rate": 3.4697853805569696e-06,
      "logits/chosen": -1.6021792888641357,
      "logits/rejected": -1.6169652938842773,
      "logps/chosen": -155.7438201904297,
      "logps/rejected": -232.19357299804688,
      "loss": 0.1113,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7856390476226807,
      "rewards/margins": 6.977983474731445,
      "rewards/rejected": -8.763622283935547,
      "step": 1920
    },
    {
      "epoch": 2.51,
      "learning_rate": 3.4515967356249263e-06,
      "logits/chosen": -1.7968169450759888,
      "logits/rejected": -1.7696614265441895,
      "logps/chosen": -164.24273681640625,
      "logps/rejected": -259.5146484375,
      "loss": 0.0443,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3350324630737305,
      "rewards/margins": 8.025506019592285,
      "rewards/rejected": -9.3605375289917,
      "step": 1921
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.4334523529611416e-06,
      "logits/chosen": -1.7174710035324097,
      "logits/rejected": -1.7741576433181763,
      "logps/chosen": -136.82872009277344,
      "logps/rejected": -225.63967895507812,
      "loss": 0.0535,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2529007196426392,
      "rewards/margins": 7.269378185272217,
      "rewards/rejected": -8.52227783203125,
      "step": 1922
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.415352269835731e-06,
      "logits/chosen": -1.9516363143920898,
      "logits/rejected": -1.895339846611023,
      "logps/chosen": -197.3709716796875,
      "logps/rejected": -279.474609375,
      "loss": 0.047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7765395641326904,
      "rewards/margins": 9.635936737060547,
      "rewards/rejected": -10.4124755859375,
      "step": 1923
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.3972965234278065e-06,
      "logits/chosen": -1.8225760459899902,
      "logits/rejected": -1.8396235704421997,
      "logps/chosen": -136.7786407470703,
      "logps/rejected": -222.9643096923828,
      "loss": 0.0021,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3355761766433716,
      "rewards/margins": 7.726853847503662,
      "rewards/rejected": -9.062429428100586,
      "step": 1924
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.379285150825434e-06,
      "logits/chosen": -1.6388682126998901,
      "logits/rejected": -1.6160069704055786,
      "logps/chosen": -175.06793212890625,
      "logps/rejected": -254.59640502929688,
      "loss": 0.0882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.813652753829956,
      "rewards/margins": 8.911599159240723,
      "rewards/rejected": -10.725251197814941,
      "step": 1925
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.3613181890255056e-06,
      "logits/chosen": -1.8454397916793823,
      "logits/rejected": -1.8451273441314697,
      "logps/chosen": -165.23707580566406,
      "logps/rejected": -253.57745361328125,
      "loss": 0.0033,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4948103427886963,
      "rewards/margins": 8.175582885742188,
      "rewards/rejected": -9.670392990112305,
      "step": 1926
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.343395674933711e-06,
      "logits/chosen": -1.8392910957336426,
      "logits/rejected": -1.8554747104644775,
      "logps/chosen": -159.6482696533203,
      "logps/rejected": -257.08721923828125,
      "loss": 0.0595,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.441204309463501,
      "rewards/margins": 8.915895462036133,
      "rewards/rejected": -10.357100486755371,
      "step": 1927
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.325517645364429e-06,
      "logits/chosen": -1.8267394304275513,
      "logits/rejected": -1.8705501556396484,
      "logps/chosen": -157.94894409179688,
      "logps/rejected": -278.55889892578125,
      "loss": 0.0601,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1422017812728882,
      "rewards/margins": 10.427032470703125,
      "rewards/rejected": -11.569234848022461,
      "step": 1928
    },
    {
      "epoch": 2.52,
      "learning_rate": 3.3076841370406674e-06,
      "logits/chosen": -2.0515806674957275,
      "logits/rejected": -1.9411430358886719,
      "logps/chosen": -146.053955078125,
      "logps/rejected": -207.13174438476562,
      "loss": 0.0477,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9206551909446716,
      "rewards/margins": 7.86514949798584,
      "rewards/rejected": -8.785804748535156,
      "step": 1929
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.289895186593972e-06,
      "logits/chosen": -1.6975440979003906,
      "logits/rejected": -1.6842153072357178,
      "logps/chosen": -159.93817138671875,
      "logps/rejected": -235.24234008789062,
      "loss": 0.136,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.1553494930267334,
      "rewards/margins": 7.111508846282959,
      "rewards/rejected": -9.26685905456543,
      "step": 1930
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.27215083056438e-06,
      "logits/chosen": -1.4588871002197266,
      "logits/rejected": -1.4719256162643433,
      "logps/chosen": -171.8911895751953,
      "logps/rejected": -271.215576171875,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4915062189102173,
      "rewards/margins": 8.634300231933594,
      "rewards/rejected": -10.125805854797363,
      "step": 1931
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.2544511054003246e-06,
      "logits/chosen": -1.708709955215454,
      "logits/rejected": -1.8296911716461182,
      "logps/chosen": -170.82650756835938,
      "logps/rejected": -288.9659118652344,
      "loss": 0.0011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.3941035270690918,
      "rewards/margins": 10.126511573791504,
      "rewards/rejected": -11.520614624023438,
      "step": 1932
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.2367960474585458e-06,
      "logits/chosen": -1.8430719375610352,
      "logits/rejected": -1.8180656433105469,
      "logps/chosen": -159.85675048828125,
      "logps/rejected": -239.32125854492188,
      "loss": 0.0882,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.850249171257019,
      "rewards/margins": 8.184515953063965,
      "rewards/rejected": -10.034765243530273,
      "step": 1933
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.2191856930040646e-06,
      "logits/chosen": -1.542874813079834,
      "logits/rejected": -1.560692310333252,
      "logps/chosen": -184.4796142578125,
      "logps/rejected": -266.9275207519531,
      "loss": 0.0462,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.3079640865325928,
      "rewards/margins": 7.735645294189453,
      "rewards/rejected": -10.043609619140625,
      "step": 1934
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.2016200782100436e-06,
      "logits/chosen": -1.7718005180358887,
      "logits/rejected": -1.7597206830978394,
      "logps/chosen": -169.38258361816406,
      "logps/rejected": -239.42462158203125,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.42669862508773804,
      "rewards/margins": 7.591115474700928,
      "rewards/rejected": -8.017814636230469,
      "step": 1935
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.18409923915777e-06,
      "logits/chosen": -1.8166931867599487,
      "logits/rejected": -1.7885932922363281,
      "logps/chosen": -190.51512145996094,
      "logps/rejected": -269.60870361328125,
      "loss": 0.0457,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.9867885112762451,
      "rewards/margins": 7.9365715980529785,
      "rewards/rejected": -9.923360824584961,
      "step": 1936
    },
    {
      "epoch": 2.53,
      "learning_rate": 3.1666232118365474e-06,
      "logits/chosen": -1.9509955644607544,
      "logits/rejected": -1.9135180711746216,
      "logps/chosen": -166.87698364257812,
      "logps/rejected": -239.43319702148438,
      "loss": 0.0878,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2764252424240112,
      "rewards/margins": 8.048609733581543,
      "rewards/rejected": -9.325035095214844,
      "step": 1937
    },
    {
      "epoch": 2.54,
      "learning_rate": 3.1491920321436303e-06,
      "logits/chosen": -1.6175456047058105,
      "logits/rejected": -1.6376662254333496,
      "logps/chosen": -182.52845764160156,
      "logps/rejected": -251.5915985107422,
      "loss": 0.0447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.605447769165039,
      "rewards/margins": 7.5432281494140625,
      "rewards/rejected": -9.148675918579102,
      "step": 1938
    },
    {
      "epoch": 2.54,
      "learning_rate": 3.1318057358841745e-06,
      "logits/chosen": -1.6363654136657715,
      "logits/rejected": -1.647525668144226,
      "logps/chosen": -160.9771270751953,
      "logps/rejected": -269.25146484375,
      "loss": 0.0886,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.8854254484176636,
      "rewards/margins": 7.888744354248047,
      "rewards/rejected": -8.774169921875,
      "step": 1939
    },
    {
      "epoch": 2.54,
      "learning_rate": 3.114464358771102e-06,
      "logits/chosen": -1.8442357778549194,
      "logits/rejected": -1.8936015367507935,
      "logps/chosen": -175.939453125,
      "logps/rejected": -274.478515625,
      "loss": 0.0871,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.350079894065857,
      "rewards/margins": 7.957852840423584,
      "rewards/rejected": -9.30793285369873,
      "step": 1940
    },
    {
      "epoch": 2.54,
      "learning_rate": 3.097167936425094e-06,
      "logits/chosen": -1.9440687894821167,
      "logits/rejected": -2.0133514404296875,
      "logps/chosen": -135.95526123046875,
      "logps/rejected": -238.60787963867188,
      "loss": 0.1314,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.7009179592132568,
      "rewards/margins": 8.161484718322754,
      "rewards/rejected": -9.86240291595459,
      "step": 1941
    },
    {
      "epoch": 2.54,
      "learning_rate": 3.079916504374494e-06,
      "logits/chosen": -1.4480254650115967,
      "logits/rejected": -1.5185046195983887,
      "logps/chosen": -156.48655700683594,
      "logps/rejected": -229.46835327148438,
      "loss": 0.0904,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -2.043402671813965,
      "rewards/margins": 7.545105934143066,
      "rewards/rejected": -9.588509559631348,
      "step": 1942
    },
    {
      "epoch": 2.54,
      "learning_rate": 3.0627100980552133e-06,
      "logits/chosen": -1.8877803087234497,
      "logits/rejected": -1.8532047271728516,
      "logps/chosen": -155.04400634765625,
      "logps/rejected": -254.82266235351562,
      "loss": 0.0477,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6466442346572876,
      "rewards/margins": 9.363658905029297,
      "rewards/rejected": -10.01030445098877,
      "step": 1943
    },
    {
      "epoch": 2.54,
      "learning_rate": 3.045548752810687e-06,
      "logits/chosen": -1.7403494119644165,
      "logits/rejected": -1.7586822509765625,
      "logps/chosen": -179.17599487304688,
      "logps/rejected": -280.99530029296875,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0659478902816772,
      "rewards/margins": 9.588112831115723,
      "rewards/rejected": -10.654060363769531,
      "step": 1944
    },
    {
      "epoch": 2.55,
      "learning_rate": 3.028432503891801e-06,
      "logits/chosen": -1.9094159603118896,
      "logits/rejected": -1.85126793384552,
      "logps/chosen": -168.78428649902344,
      "logps/rejected": -244.7453155517578,
      "loss": 0.0445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7366176247596741,
      "rewards/margins": 8.83923053741455,
      "rewards/rejected": -9.575847625732422,
      "step": 1945
    },
    {
      "epoch": 2.55,
      "learning_rate": 3.01136138645679e-06,
      "logits/chosen": -1.8410999774932861,
      "logits/rejected": -1.9505757093429565,
      "logps/chosen": -151.09922790527344,
      "logps/rejected": -261.0325622558594,
      "loss": 0.0437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.48792317509651184,
      "rewards/margins": 9.749568939208984,
      "rewards/rejected": -10.237492561340332,
      "step": 1946
    },
    {
      "epoch": 2.55,
      "learning_rate": 2.9943354355711884e-06,
      "logits/chosen": -1.6905401945114136,
      "logits/rejected": -1.7848055362701416,
      "logps/chosen": -165.78990173339844,
      "logps/rejected": -287.92071533203125,
      "loss": 0.0869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1763222217559814,
      "rewards/margins": 9.303790092468262,
      "rewards/rejected": -10.480112075805664,
      "step": 1947
    },
    {
      "epoch": 2.55,
      "learning_rate": 2.9773546862077617e-06,
      "logits/chosen": -1.814382791519165,
      "logits/rejected": -1.834534764289856,
      "logps/chosen": -202.5555419921875,
      "logps/rejected": -301.493896484375,
      "loss": 0.0466,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2390787601470947,
      "rewards/margins": 9.512657165527344,
      "rewards/rejected": -10.75173568725586,
      "step": 1948
    },
    {
      "epoch": 2.55,
      "learning_rate": 2.960419173246437e-06,
      "logits/chosen": -1.6848957538604736,
      "logits/rejected": -1.6994730234146118,
      "logps/chosen": -230.0084991455078,
      "logps/rejected": -314.99871826171875,
      "loss": 0.0878,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4034273624420166,
      "rewards/margins": 8.364229202270508,
      "rewards/rejected": -9.767656326293945,
      "step": 1949
    },
    {
      "epoch": 2.55,
      "learning_rate": 2.9435289314742015e-06,
      "logits/chosen": -1.8944004774093628,
      "logits/rejected": -1.9067261219024658,
      "logps/chosen": -166.98733520507812,
      "logps/rejected": -231.96133422851562,
      "loss": 0.0458,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0498709678649902,
      "rewards/margins": 7.810283660888672,
      "rewards/rejected": -8.86015510559082,
      "step": 1950
    },
    {
      "epoch": 2.55,
      "learning_rate": 2.926683995585053e-06,
      "logits/chosen": -1.9160047769546509,
      "logits/rejected": -1.9514662027359009,
      "logps/chosen": -162.69851684570312,
      "logps/rejected": -244.1333465576172,
      "loss": 0.0479,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.8319509029388428,
      "rewards/margins": 7.391755104064941,
      "rewards/rejected": -8.223706245422363,
      "step": 1951
    },
    {
      "epoch": 2.55,
      "learning_rate": 2.9098844001799407e-06,
      "logits/chosen": -1.6513445377349854,
      "logits/rejected": -1.5727030038833618,
      "logps/chosen": -192.17137145996094,
      "logps/rejected": -286.82012939453125,
      "loss": 0.0461,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0699559450149536,
      "rewards/margins": 8.417442321777344,
      "rewards/rejected": -9.48740005493164,
      "step": 1952
    },
    {
      "epoch": 2.56,
      "learning_rate": 2.8931301797666844e-06,
      "logits/chosen": -1.794509768486023,
      "logits/rejected": -1.8245161771774292,
      "logps/chosen": -155.76063537597656,
      "logps/rejected": -244.39767456054688,
      "loss": 0.0101,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0269020795822144,
      "rewards/margins": 8.800419807434082,
      "rewards/rejected": -9.827322959899902,
      "step": 1953
    },
    {
      "epoch": 2.56,
      "learning_rate": 2.8764213687598713e-06,
      "logits/chosen": -1.8388452529907227,
      "logits/rejected": -1.8234366178512573,
      "logps/chosen": -197.64418029785156,
      "logps/rejected": -312.91436767578125,
      "loss": 0.0437,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0774991512298584,
      "rewards/margins": 10.790443420410156,
      "rewards/rejected": -12.867941856384277,
      "step": 1954
    },
    {
      "epoch": 2.56,
      "learning_rate": 2.85975800148085e-06,
      "logits/chosen": -1.6822351217269897,
      "logits/rejected": -1.7702974081039429,
      "logps/chosen": -177.27000427246094,
      "logps/rejected": -298.2420959472656,
      "loss": 0.0008,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0947136878967285,
      "rewards/margins": 9.989084243774414,
      "rewards/rejected": -11.0837984085083,
      "step": 1955
    },
    {
      "epoch": 2.56,
      "learning_rate": 2.843140112157594e-06,
      "logits/chosen": -1.8989745378494263,
      "logits/rejected": -1.8452208042144775,
      "logps/chosen": -146.25962829589844,
      "logps/rejected": -210.71157836914062,
      "loss": 0.0049,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4391261339187622,
      "rewards/margins": 7.164846420288086,
      "rewards/rejected": -8.603972434997559,
      "step": 1956
    },
    {
      "epoch": 2.56,
      "learning_rate": 2.8265677349246735e-06,
      "logits/chosen": -1.88750159740448,
      "logits/rejected": -1.8424099683761597,
      "logps/chosen": -177.20985412597656,
      "logps/rejected": -265.3443908691406,
      "loss": 0.0446,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6870248317718506,
      "rewards/margins": 8.107356071472168,
      "rewards/rejected": -9.794380187988281,
      "step": 1957
    },
    {
      "epoch": 2.56,
      "learning_rate": 2.8100409038231746e-06,
      "logits/chosen": -1.9545717239379883,
      "logits/rejected": -1.985085129737854,
      "logps/chosen": -174.42807006835938,
      "logps/rejected": -243.71665954589844,
      "loss": 0.0467,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.422469139099121,
      "rewards/margins": 7.641671180725098,
      "rewards/rejected": -9.064140319824219,
      "step": 1958
    },
    {
      "epoch": 2.56,
      "learning_rate": 2.793559652800631e-06,
      "logits/chosen": -2.0010335445404053,
      "logits/rejected": -2.0178062915802,
      "logps/chosen": -197.19537353515625,
      "logps/rejected": -257.2098693847656,
      "loss": 0.0912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.4117121696472168,
      "rewards/margins": 6.950974464416504,
      "rewards/rejected": -8.362686157226562,
      "step": 1959
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.7771240157109355e-06,
      "logits/chosen": -1.6148195266723633,
      "logits/rejected": -1.613033652305603,
      "logps/chosen": -185.54635620117188,
      "logps/rejected": -277.41259765625,
      "loss": 0.0018,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.2018530368804932,
      "rewards/margins": 9.391947746276855,
      "rewards/rejected": -10.59380054473877,
      "step": 1960
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.7607340263143073e-06,
      "logits/chosen": -1.6373337507247925,
      "logits/rejected": -1.6041699647903442,
      "logps/chosen": -176.4813232421875,
      "logps/rejected": -254.50308227539062,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.347382068634033,
      "rewards/margins": 7.4096879959106445,
      "rewards/rejected": -9.75706958770752,
      "step": 1961
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.7443897182771794e-06,
      "logits/chosen": -1.7655137777328491,
      "logits/rejected": -1.7664759159088135,
      "logps/chosen": -203.99636840820312,
      "logps/rejected": -297.6118469238281,
      "loss": 0.0024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0905452966690063,
      "rewards/margins": 9.155332565307617,
      "rewards/rejected": -10.245878219604492,
      "step": 1962
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.7280911251721748e-06,
      "logits/chosen": -1.922075867652893,
      "logits/rejected": -1.9698618650436401,
      "logps/chosen": -136.09805297851562,
      "logps/rejected": -220.62811279296875,
      "loss": 0.0454,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5097430348396301,
      "rewards/margins": 8.091923713684082,
      "rewards/rejected": -8.601667404174805,
      "step": 1963
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.711838280477988e-06,
      "logits/chosen": -1.8058196306228638,
      "logits/rejected": -1.7713044881820679,
      "logps/chosen": -139.84765625,
      "logps/rejected": -242.8928680419922,
      "loss": 0.001,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4930356740951538,
      "rewards/margins": 10.188127517700195,
      "rewards/rejected": -10.68116283416748,
      "step": 1964
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.6956312175793613e-06,
      "logits/chosen": -2.073621988296509,
      "logits/rejected": -2.022918224334717,
      "logps/chosen": -168.22671508789062,
      "logps/rejected": -275.7758483886719,
      "loss": 0.0075,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.4368441104888916,
      "rewards/margins": 11.374152183532715,
      "rewards/rejected": -11.810997009277344,
      "step": 1965
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.679469969767001e-06,
      "logits/chosen": -1.8209521770477295,
      "logits/rejected": -1.8305972814559937,
      "logps/chosen": -139.6355743408203,
      "logps/rejected": -250.3299560546875,
      "loss": 0.0436,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7493610382080078,
      "rewards/margins": 9.984441757202148,
      "rewards/rejected": -10.733802795410156,
      "step": 1966
    },
    {
      "epoch": 2.57,
      "learning_rate": 2.663354570237481e-06,
      "logits/chosen": -1.9432001113891602,
      "logits/rejected": -1.9751665592193604,
      "logps/chosen": -186.65890502929688,
      "logps/rejected": -281.393310546875,
      "loss": 0.0443,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.459289312362671,
      "rewards/margins": 8.835786819458008,
      "rewards/rejected": -10.295076370239258,
      "step": 1967
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.647285052093218e-06,
      "logits/chosen": -1.6783727407455444,
      "logits/rejected": -1.663486361503601,
      "logps/chosen": -219.9584197998047,
      "logps/rejected": -323.7277526855469,
      "loss": 0.0453,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.44733792543411255,
      "rewards/margins": 10.029692649841309,
      "rewards/rejected": -10.477030754089355,
      "step": 1968
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.631261448342387e-06,
      "logits/chosen": -1.875363826751709,
      "logits/rejected": -1.8750375509262085,
      "logps/chosen": -176.53768920898438,
      "logps/rejected": -267.9600524902344,
      "loss": 0.0002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.6982587575912476,
      "rewards/margins": 10.102518081665039,
      "rewards/rejected": -10.800776481628418,
      "step": 1969
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.615283791898837e-06,
      "logits/chosen": -1.710947871208191,
      "logits/rejected": -1.8196959495544434,
      "logps/chosen": -150.7943572998047,
      "logps/rejected": -277.4522399902344,
      "loss": 0.0044,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.4654209613800049,
      "rewards/margins": 10.907736778259277,
      "rewards/rejected": -12.37315845489502,
      "step": 1970
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.599352115582046e-06,
      "logits/chosen": -1.9294018745422363,
      "logits/rejected": -1.984405755996704,
      "logps/chosen": -154.47694396972656,
      "logps/rejected": -261.33673095703125,
      "loss": 0.003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1193416118621826,
      "rewards/margins": 9.98946762084961,
      "rewards/rejected": -11.108809471130371,
      "step": 1971
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.5834664521170504e-06,
      "logits/chosen": -1.9025990962982178,
      "logits/rejected": -1.8562264442443848,
      "logps/chosen": -172.32485961914062,
      "logps/rejected": -259.0001525878906,
      "loss": 0.0006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2798805832862854,
      "rewards/margins": 11.349750518798828,
      "rewards/rejected": -11.069870948791504,
      "step": 1972
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.5676268341343622e-06,
      "logits/chosen": -1.928761601448059,
      "logits/rejected": -1.88590669631958,
      "logps/chosen": -149.4075927734375,
      "logps/rejected": -214.0815887451172,
      "loss": 0.0455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.6468234062194824,
      "rewards/margins": 7.01922607421875,
      "rewards/rejected": -8.66604995727539,
      "step": 1973
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.5518332941699056e-06,
      "logits/chosen": -2.022953987121582,
      "logits/rejected": -2.0281288623809814,
      "logps/chosen": -213.13470458984375,
      "logps/rejected": -249.05712890625,
      "loss": 0.1758,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -2.019098997116089,
      "rewards/margins": 6.2226433753967285,
      "rewards/rejected": -8.241742134094238,
      "step": 1974
    },
    {
      "epoch": 2.58,
      "learning_rate": 2.5360858646649722e-06,
      "logits/chosen": -1.5471394062042236,
      "logits/rejected": -1.5764353275299072,
      "logps/chosen": -177.44534301757812,
      "logps/rejected": -251.7624053955078,
      "loss": 0.0882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.3179144859313965,
      "rewards/margins": 6.850772857666016,
      "rewards/rejected": -9.16868782043457,
      "step": 1975
    },
    {
      "epoch": 2.59,
      "learning_rate": 2.520384577966142e-06,
      "logits/chosen": -1.638532042503357,
      "logits/rejected": -1.5607911348342896,
      "logps/chosen": -191.2447509765625,
      "logps/rejected": -315.3387756347656,
      "loss": 0.0003,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.9296466708183289,
      "rewards/margins": 11.583292961120605,
      "rewards/rejected": -12.512941360473633,
      "step": 1976
    },
    {
      "epoch": 2.59,
      "learning_rate": 2.5047294663251953e-06,
      "logits/chosen": -1.693674087524414,
      "logits/rejected": -1.6914783716201782,
      "logps/chosen": -180.8939208984375,
      "logps/rejected": -277.2292785644531,
      "loss": 0.0023,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.7101593017578125,
      "rewards/margins": 10.458864212036133,
      "rewards/rejected": -11.169023513793945,
      "step": 1977
    },
    {
      "epoch": 2.59,
      "learning_rate": 2.4891205618990666e-06,
      "logits/chosen": -1.7112553119659424,
      "logits/rejected": -1.6535606384277344,
      "logps/chosen": -210.1945343017578,
      "logps/rejected": -274.9900207519531,
      "loss": 0.0901,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.835531234741211,
      "rewards/margins": 7.511919021606445,
      "rewards/rejected": -9.347450256347656,
      "step": 1978
    },
    {
      "epoch": 2.59,
      "learning_rate": 2.4735578967497953e-06,
      "logits/chosen": -1.9723058938980103,
      "logits/rejected": -1.9816824197769165,
      "logps/chosen": -182.5241241455078,
      "logps/rejected": -238.19497680664062,
      "loss": 0.0503,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.207892894744873,
      "rewards/margins": 7.347690105438232,
      "rewards/rejected": -9.555582046508789,
      "step": 1979
    },
    {
      "epoch": 2.59,
      "learning_rate": 2.4580415028444326e-06,
      "logits/chosen": -1.9334659576416016,
      "logits/rejected": -1.9646167755126953,
      "logps/chosen": -251.38478088378906,
      "logps/rejected": -332.16796875,
      "loss": 0.1306,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.3673992156982422,
      "rewards/margins": 7.479816436767578,
      "rewards/rejected": -8.847214698791504,
      "step": 1980
    },
    {
      "epoch": 2.59,
      "learning_rate": 2.4425714120549726e-06,
      "logits/chosen": -1.9385401010513306,
      "logits/rejected": -1.9572641849517822,
      "logps/chosen": -154.16470336914062,
      "logps/rejected": -271.7256774902344,
      "loss": 0.0444,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9760931730270386,
      "rewards/margins": 9.893794059753418,
      "rewards/rejected": -10.869888305664062,
      "step": 1981
    },
    {
      "epoch": 2.59,
      "learning_rate": 2.42714765615831e-06,
      "logits/chosen": -1.957564115524292,
      "logits/rejected": -1.954709768295288,
      "logps/chosen": -195.2952423095703,
      "logps/rejected": -257.8908386230469,
      "loss": 0.0219,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.0064735412597656,
      "rewards/margins": 8.165589332580566,
      "rewards/rejected": -10.172062873840332,
      "step": 1982
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.4117702668361777e-06,
      "logits/chosen": -1.6143949031829834,
      "logits/rejected": -1.5937446355819702,
      "logps/chosen": -211.5524139404297,
      "logps/rejected": -265.3187561035156,
      "loss": 0.0877,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.158461570739746,
      "rewards/margins": 6.327495574951172,
      "rewards/rejected": -8.485957145690918,
      "step": 1983
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.3964392756750276e-06,
      "logits/chosen": -1.7104253768920898,
      "logits/rejected": -1.7116541862487793,
      "logps/chosen": -161.94729614257812,
      "logps/rejected": -231.95826721191406,
      "loss": 0.0889,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.1934306621551514,
      "rewards/margins": 7.831337928771973,
      "rewards/rejected": -9.024768829345703,
      "step": 1984
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.381154714166045e-06,
      "logits/chosen": -1.7671799659729004,
      "logits/rejected": -1.744809865951538,
      "logps/chosen": -185.0078125,
      "logps/rejected": -247.1660614013672,
      "loss": 0.049,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.276573657989502,
      "rewards/margins": 7.445742607116699,
      "rewards/rejected": -8.722315788269043,
      "step": 1985
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.3659166137050297e-06,
      "logits/chosen": -1.7502305507659912,
      "logits/rejected": -1.696411371231079,
      "logps/chosen": -205.7821502685547,
      "logps/rejected": -265.5729675292969,
      "loss": 0.0032,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.1911041736602783,
      "rewards/margins": 8.429696083068848,
      "rewards/rejected": -10.620800971984863,
      "step": 1986
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.3507250055923384e-06,
      "logits/chosen": -1.4697926044464111,
      "logits/rejected": -1.5125470161437988,
      "logps/chosen": -145.3203582763672,
      "logps/rejected": -234.50877380371094,
      "loss": 0.0449,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.3017122745513916,
      "rewards/margins": 8.171239852905273,
      "rewards/rejected": -9.472951889038086,
      "step": 1987
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.335579921032849e-06,
      "logits/chosen": -1.783079981803894,
      "logits/rejected": -1.73946213722229,
      "logps/chosen": -189.13800048828125,
      "logps/rejected": -279.6351318359375,
      "loss": 0.0123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.1208068132400513,
      "rewards/margins": 8.408231735229492,
      "rewards/rejected": -9.529037475585938,
      "step": 1988
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.3204813911358535e-06,
      "logits/chosen": -1.3521227836608887,
      "logits/rejected": -1.326317548751831,
      "logps/chosen": -165.36524963378906,
      "logps/rejected": -251.8580322265625,
      "loss": 0.0898,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.272942304611206,
      "rewards/margins": 7.702919960021973,
      "rewards/rejected": -8.975862503051758,
      "step": 1989
    },
    {
      "epoch": 2.6,
      "learning_rate": 2.305429446915036e-06,
      "logits/chosen": -1.9946447610855103,
      "logits/rejected": -2.0219244956970215,
      "logps/chosen": -147.47792053222656,
      "logps/rejected": -244.192138671875,
      "loss": 0.0885,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7088050842285156,
      "rewards/margins": 8.978840827941895,
      "rewards/rejected": -9.687644958496094,
      "step": 1990
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.2904241192883703e-06,
      "logits/chosen": -1.753751516342163,
      "logits/rejected": -1.778114676475525,
      "logps/chosen": -155.39047241210938,
      "logps/rejected": -235.30279541015625,
      "loss": 0.0477,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.966157078742981,
      "rewards/margins": 8.723522186279297,
      "rewards/rejected": -9.689678192138672,
      "step": 1991
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.2754654390780924e-06,
      "logits/chosen": -1.6753144264221191,
      "logits/rejected": -1.6919025182724,
      "logps/chosen": -185.6756591796875,
      "logps/rejected": -272.6050720214844,
      "loss": 0.1308,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.3676074743270874,
      "rewards/margins": 8.392903327941895,
      "rewards/rejected": -9.760510444641113,
      "step": 1992
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.260553437010621e-06,
      "logits/chosen": -1.8272124528884888,
      "logits/rejected": -1.7892577648162842,
      "logps/chosen": -162.16757202148438,
      "logps/rejected": -252.224365234375,
      "loss": 0.002,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.6557848453521729,
      "rewards/margins": 8.761106491088867,
      "rewards/rejected": -10.416891098022461,
      "step": 1993
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.245688143716476e-06,
      "logits/chosen": -1.6903076171875,
      "logits/rejected": -1.7331631183624268,
      "logps/chosen": -129.78890991210938,
      "logps/rejected": -253.65911865234375,
      "loss": 0.0475,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4502795934677124,
      "rewards/margins": 10.536611557006836,
      "rewards/rejected": -10.98689079284668,
      "step": 1994
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.2308695897302472e-06,
      "logits/chosen": -1.8485065698623657,
      "logits/rejected": -1.8725085258483887,
      "logps/chosen": -173.86074829101562,
      "logps/rejected": -274.06280517578125,
      "loss": 0.0873,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -1.8706365823745728,
      "rewards/margins": 9.682866096496582,
      "rewards/rejected": -11.553502082824707,
      "step": 1995
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.216097805490516e-06,
      "logits/chosen": -1.7529674768447876,
      "logits/rejected": -1.7482428550720215,
      "logps/chosen": -172.43064880371094,
      "logps/rejected": -268.3185119628906,
      "loss": 0.0014,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.7001947164535522,
      "rewards/margins": 9.211479187011719,
      "rewards/rejected": -10.911673545837402,
      "step": 1996
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.2013728213398006e-06,
      "logits/chosen": -1.603108525276184,
      "logits/rejected": -1.5340635776519775,
      "logps/chosen": -158.3709716796875,
      "logps/rejected": -266.3359069824219,
      "loss": 0.0439,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.525465726852417,
      "rewards/margins": 10.82684326171875,
      "rewards/rejected": -12.352310180664062,
      "step": 1997
    },
    {
      "epoch": 2.61,
      "learning_rate": 2.1866946675244692e-06,
      "logits/chosen": -1.8880060911178589,
      "logits/rejected": -1.888407826423645,
      "logps/chosen": -164.61329650878906,
      "logps/rejected": -249.24034118652344,
      "loss": 0.0465,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.0754520893096924,
      "rewards/margins": 8.30090045928955,
      "rewards/rejected": -9.37635326385498,
      "step": 1998
    },
    {
      "epoch": 2.62,
      "learning_rate": 2.1720633741947187e-06,
      "logits/chosen": -1.951326847076416,
      "logits/rejected": -1.9635450839996338,
      "logps/chosen": -162.000732421875,
      "logps/rejected": -238.7664337158203,
      "loss": 0.0438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6137329936027527,
      "rewards/margins": 8.497785568237305,
      "rewards/rejected": -9.111517906188965,
      "step": 1999
    },
    {
      "epoch": 2.62,
      "learning_rate": 2.157478971404478e-06,
      "logits/chosen": -2.010573148727417,
      "logits/rejected": -1.9262150526046753,
      "logps/chosen": -239.3760986328125,
      "logps/rejected": -321.3377685546875,
      "loss": 0.1308,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -1.657750129699707,
      "rewards/margins": 7.968457221984863,
      "rewards/rejected": -9.62620735168457,
      "step": 2000
    }
  ],
  "logging_steps": 1,
  "max_steps": 2292,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}