{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1563,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 43.373348236083984,
      "kl": 0.018903231248259544,
      "learning_rate": 1.592356687898089e-08,
      "logps/chosen": -299.6726379394531,
      "logps/rejected": -346.7806396484375,
      "loss": 0.5001,
      "rewards/chosen": -0.0007079701172187924,
      "rewards/margins": -0.001962849870324135,
      "rewards/rejected": 0.0012548796366900206,
      "step": 5
    },
    {
      "epoch": 0.01,
      "grad_norm": 45.312686920166016,
      "kl": 0.05140366405248642,
      "learning_rate": 3.184713375796178e-08,
      "logps/chosen": -310.0472717285156,
      "logps/rejected": -311.03424072265625,
      "loss": 0.4979,
      "rewards/chosen": 0.019934307783842087,
      "rewards/margins": 0.010924299247562885,
      "rewards/rejected": 0.009010007604956627,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 44.28553771972656,
      "kl": 0.01275167427957058,
      "learning_rate": 4.777070063694268e-08,
      "logps/chosen": -258.36212158203125,
      "logps/rejected": -348.38824462890625,
      "loss": 0.4943,
      "rewards/chosen": 0.04069889709353447,
      "rewards/margins": 0.04853983223438263,
      "rewards/rejected": -0.007840934209525585,
      "step": 15
    },
    {
      "epoch": 0.01,
      "grad_norm": 45.054351806640625,
      "kl": 0.0,
      "learning_rate": 6.369426751592356e-08,
      "logps/chosen": -245.97470092773438,
      "logps/rejected": -350.9342041015625,
      "loss": 0.4837,
      "rewards/chosen": 0.07507207244634628,
      "rewards/margins": 0.13315154612064362,
      "rewards/rejected": -0.05807947367429733,
      "step": 20
    },
    {
      "epoch": 0.02,
      "grad_norm": 43.90499496459961,
      "kl": 0.0,
      "learning_rate": 7.961783439490445e-08,
      "logps/chosen": -240.2781524658203,
      "logps/rejected": -334.9874267578125,
      "loss": 0.465,
      "rewards/chosen": 0.16415421664714813,
      "rewards/margins": 0.27640852332115173,
      "rewards/rejected": -0.11225433647632599,
      "step": 25
    },
    {
      "epoch": 0.02,
      "grad_norm": 42.99668884277344,
      "kl": 0.0,
      "learning_rate": 9.554140127388536e-08,
      "logps/chosen": -310.78314208984375,
      "logps/rejected": -325.97113037109375,
      "loss": 0.4387,
      "rewards/chosen": 0.21505948901176453,
      "rewards/margins": 0.4358912408351898,
      "rewards/rejected": -0.22083178162574768,
      "step": 30
    },
    {
      "epoch": 0.02,
      "grad_norm": 42.578617095947266,
      "kl": 0.0,
      "learning_rate": 1.1146496815286624e-07,
      "logps/chosen": -263.4063415527344,
      "logps/rejected": -312.7994689941406,
      "loss": 0.4157,
      "rewards/chosen": 0.25127312541007996,
      "rewards/margins": 0.7119064331054688,
      "rewards/rejected": -0.46063321828842163,
      "step": 35
    },
    {
      "epoch": 0.03,
      "grad_norm": 38.7848014831543,
      "kl": 0.0,
      "learning_rate": 1.2738853503184713e-07,
      "logps/chosen": -283.001953125,
      "logps/rejected": -342.24493408203125,
      "loss": 0.3764,
      "rewards/chosen": 0.18532009422779083,
      "rewards/margins": 1.1129443645477295,
      "rewards/rejected": -0.9276243448257446,
      "step": 40
    },
    {
      "epoch": 0.03,
      "grad_norm": 33.522525787353516,
      "kl": 0.0,
      "learning_rate": 1.43312101910828e-07,
      "logps/chosen": -263.763916015625,
      "logps/rejected": -356.7685852050781,
      "loss": 0.3194,
      "rewards/chosen": 0.22150889039039612,
      "rewards/margins": 1.852413535118103,
      "rewards/rejected": -1.6309047937393188,
      "step": 45
    },
    {
      "epoch": 0.03,
      "grad_norm": 34.30215072631836,
      "kl": 0.0,
      "learning_rate": 1.592356687898089e-07,
      "logps/chosen": -305.10076904296875,
      "logps/rejected": -346.50970458984375,
      "loss": 0.3089,
      "rewards/chosen": 0.23655760288238525,
      "rewards/margins": 2.413478136062622,
      "rewards/rejected": -2.1769204139709473,
      "step": 50
    },
    {
      "epoch": 0.04,
      "grad_norm": 63.321285247802734,
      "kl": 0.0,
      "learning_rate": 1.7515923566878978e-07,
      "logps/chosen": -304.7918701171875,
      "logps/rejected": -375.59271240234375,
      "loss": 0.289,
      "rewards/chosen": 0.26213210821151733,
      "rewards/margins": 2.895750045776367,
      "rewards/rejected": -2.633617877960205,
      "step": 55
    },
    {
      "epoch": 0.04,
      "grad_norm": 31.712352752685547,
      "kl": 0.0,
      "learning_rate": 1.9108280254777072e-07,
      "logps/chosen": -261.5682067871094,
      "logps/rejected": -345.7394104003906,
      "loss": 0.2843,
      "rewards/chosen": 0.3677944540977478,
      "rewards/margins": 2.7658519744873047,
      "rewards/rejected": -2.398057460784912,
      "step": 60
    },
    {
      "epoch": 0.04,
      "grad_norm": 33.49573516845703,
      "kl": 0.0,
      "learning_rate": 2.070063694267516e-07,
      "logps/chosen": -300.9778137207031,
      "logps/rejected": -324.8972473144531,
      "loss": 0.2645,
      "rewards/chosen": 0.3324030041694641,
      "rewards/margins": 3.486532688140869,
      "rewards/rejected": -3.1541295051574707,
      "step": 65
    },
    {
      "epoch": 0.04,
      "grad_norm": 33.059207916259766,
      "kl": 0.0,
      "learning_rate": 2.2292993630573247e-07,
      "logps/chosen": -318.46124267578125,
      "logps/rejected": -387.02947998046875,
      "loss": 0.2496,
      "rewards/chosen": 0.3689573109149933,
      "rewards/margins": 3.851461887359619,
      "rewards/rejected": -3.482504367828369,
      "step": 70
    },
    {
      "epoch": 0.05,
      "grad_norm": 31.94024658203125,
      "kl": 0.0,
      "learning_rate": 2.388535031847134e-07,
      "logps/chosen": -358.51654052734375,
      "logps/rejected": -330.6874694824219,
      "loss": 0.2602,
      "rewards/chosen": 0.41941699385643005,
      "rewards/margins": 3.710043430328369,
      "rewards/rejected": -3.2906270027160645,
      "step": 75
    },
    {
      "epoch": 0.05,
      "grad_norm": 34.636436462402344,
      "kl": 0.0,
      "learning_rate": 2.5477707006369425e-07,
      "logps/chosen": -268.0237121582031,
      "logps/rejected": -380.11676025390625,
      "loss": 0.2606,
      "rewards/chosen": 0.410856157541275,
      "rewards/margins": 4.273175239562988,
      "rewards/rejected": -3.862318754196167,
      "step": 80
    },
    {
      "epoch": 0.05,
      "grad_norm": 29.300586700439453,
      "kl": 0.0,
      "learning_rate": 2.7070063694267513e-07,
      "logps/chosen": -323.76416015625,
      "logps/rejected": -359.8213195800781,
      "loss": 0.2378,
      "rewards/chosen": 0.46109551191329956,
      "rewards/margins": 4.482287406921387,
      "rewards/rejected": -4.021191596984863,
      "step": 85
    },
    {
      "epoch": 0.06,
      "grad_norm": 32.00096130371094,
      "kl": 0.0,
      "learning_rate": 2.86624203821656e-07,
      "logps/chosen": -292.0968322753906,
      "logps/rejected": -414.10980224609375,
      "loss": 0.2176,
      "rewards/chosen": 0.5200246572494507,
      "rewards/margins": 5.136369228363037,
      "rewards/rejected": -4.616344928741455,
      "step": 90
    },
    {
      "epoch": 0.06,
      "grad_norm": 31.802518844604492,
      "kl": 0.0,
      "learning_rate": 3.0254777070063694e-07,
      "logps/chosen": -289.4820556640625,
      "logps/rejected": -367.87030029296875,
      "loss": 0.2372,
      "rewards/chosen": 0.5198050737380981,
      "rewards/margins": 5.272117614746094,
      "rewards/rejected": -4.752312183380127,
      "step": 95
    },
    {
      "epoch": 0.06,
      "grad_norm": 37.19041061401367,
      "kl": 0.0,
      "learning_rate": 3.184713375796178e-07,
      "logps/chosen": -280.93896484375,
      "logps/rejected": -394.2848205566406,
      "loss": 0.2338,
      "rewards/chosen": 0.4976850152015686,
      "rewards/margins": 5.734658241271973,
      "rewards/rejected": -5.236973285675049,
      "step": 100
    },
    {
      "epoch": 0.07,
      "grad_norm": 26.246625900268555,
      "kl": 0.0,
      "learning_rate": 3.343949044585987e-07,
      "logps/chosen": -315.616943359375,
      "logps/rejected": -338.24346923828125,
      "loss": 0.2378,
      "rewards/chosen": 0.5364874601364136,
      "rewards/margins": 4.776383399963379,
      "rewards/rejected": -4.239895820617676,
      "step": 105
    },
    {
      "epoch": 0.07,
      "grad_norm": 34.44779586791992,
      "kl": 0.0,
      "learning_rate": 3.5031847133757957e-07,
      "logps/chosen": -253.6793975830078,
      "logps/rejected": -391.8357238769531,
      "loss": 0.2379,
      "rewards/chosen": 0.40439778566360474,
      "rewards/margins": 5.385909557342529,
      "rewards/rejected": -4.98151159286499,
      "step": 110
    },
    {
      "epoch": 0.07,
      "grad_norm": 45.468196868896484,
      "kl": 0.0,
      "learning_rate": 3.6624203821656045e-07,
      "logps/chosen": -283.26025390625,
      "logps/rejected": -404.784423828125,
      "loss": 0.2448,
      "rewards/chosen": 0.5039983987808228,
      "rewards/margins": 5.760153770446777,
      "rewards/rejected": -5.256155967712402,
      "step": 115
    },
    {
      "epoch": 0.08,
      "grad_norm": 29.123733520507812,
      "kl": 0.0,
      "learning_rate": 3.8216560509554143e-07,
      "logps/chosen": -292.062255859375,
      "logps/rejected": -350.0133056640625,
      "loss": 0.2361,
      "rewards/chosen": 0.5945883989334106,
      "rewards/margins": 5.297600746154785,
      "rewards/rejected": -4.703011989593506,
      "step": 120
    },
    {
      "epoch": 0.08,
      "grad_norm": 31.618877410888672,
      "kl": 0.0,
      "learning_rate": 3.980891719745223e-07,
      "logps/chosen": -304.79302978515625,
      "logps/rejected": -412.56396484375,
      "loss": 0.2332,
      "rewards/chosen": 0.5294879674911499,
      "rewards/margins": 5.99056339263916,
      "rewards/rejected": -5.461075782775879,
      "step": 125
    },
    {
      "epoch": 0.08,
      "grad_norm": 31.75819969177246,
      "kl": 0.0,
      "learning_rate": 4.140127388535032e-07,
      "logps/chosen": -343.9717712402344,
      "logps/rejected": -413.4139709472656,
      "loss": 0.2209,
      "rewards/chosen": 0.6387773752212524,
      "rewards/margins": 6.253381729125977,
      "rewards/rejected": -5.6146039962768555,
      "step": 130
    },
    {
      "epoch": 0.09,
      "grad_norm": 80.12673950195312,
      "kl": 0.0,
      "learning_rate": 4.2993630573248406e-07,
      "logps/chosen": -276.16864013671875,
      "logps/rejected": -370.7424621582031,
      "loss": 0.2331,
      "rewards/chosen": 0.4045650362968445,
      "rewards/margins": 5.879421710968018,
      "rewards/rejected": -5.474856853485107,
      "step": 135
    },
    {
      "epoch": 0.09,
      "grad_norm": 30.59686851501465,
      "kl": 0.0,
      "learning_rate": 4.4585987261146494e-07,
      "logps/chosen": -264.196533203125,
      "logps/rejected": -359.3782043457031,
      "loss": 0.2489,
      "rewards/chosen": 0.45963722467422485,
      "rewards/margins": 4.636446952819824,
      "rewards/rejected": -4.176809310913086,
      "step": 140
    },
    {
      "epoch": 0.09,
      "grad_norm": 28.13033103942871,
      "kl": 0.0,
      "learning_rate": 4.6178343949044587e-07,
      "logps/chosen": -269.31793212890625,
      "logps/rejected": -414.6089782714844,
      "loss": 0.2443,
      "rewards/chosen": 0.4464253783226013,
      "rewards/margins": 5.700473308563232,
      "rewards/rejected": -5.2540483474731445,
      "step": 145
    },
    {
      "epoch": 0.1,
      "grad_norm": 29.259023666381836,
      "kl": 0.0,
      "learning_rate": 4.777070063694267e-07,
      "logps/chosen": -315.52374267578125,
      "logps/rejected": -365.8375549316406,
      "loss": 0.2258,
      "rewards/chosen": 0.6224046349525452,
      "rewards/margins": 6.201630115509033,
      "rewards/rejected": -5.579225063323975,
      "step": 150
    },
    {
      "epoch": 0.1,
      "grad_norm": 32.15901565551758,
      "kl": 0.0,
      "learning_rate": 4.936305732484076e-07,
      "logps/chosen": -284.3131103515625,
      "logps/rejected": -374.58770751953125,
      "loss": 0.2226,
      "rewards/chosen": 0.5719965100288391,
      "rewards/margins": 6.714544773101807,
      "rewards/rejected": -6.142548561096191,
      "step": 155
    },
    {
      "epoch": 0.1,
      "grad_norm": 33.57276153564453,
      "kl": 0.0,
      "learning_rate": 4.989331436699858e-07,
      "logps/chosen": -291.7167053222656,
      "logps/rejected": -374.89031982421875,
      "loss": 0.2333,
      "rewards/chosen": 0.6062914133071899,
      "rewards/margins": 6.262972831726074,
      "rewards/rejected": -5.656682014465332,
      "step": 160
    },
    {
      "epoch": 0.11,
      "grad_norm": 27.332050323486328,
      "kl": 0.0,
      "learning_rate": 4.971550497866287e-07,
      "logps/chosen": -309.1024475097656,
      "logps/rejected": -402.1785583496094,
      "loss": 0.2336,
      "rewards/chosen": 0.5264068841934204,
      "rewards/margins": 6.772003173828125,
      "rewards/rejected": -6.245596408843994,
      "step": 165
    },
    {
      "epoch": 0.11,
      "grad_norm": 26.317598342895508,
      "kl": 0.0,
      "learning_rate": 4.953769559032717e-07,
      "logps/chosen": -277.711669921875,
      "logps/rejected": -364.07745361328125,
      "loss": 0.2229,
      "rewards/chosen": 0.6749172210693359,
      "rewards/margins": 6.215371131896973,
      "rewards/rejected": -5.540453910827637,
      "step": 170
    },
    {
      "epoch": 0.11,
      "grad_norm": 26.295717239379883,
      "kl": 0.0,
      "learning_rate": 4.935988620199146e-07,
      "logps/chosen": -315.84600830078125,
      "logps/rejected": -419.64947509765625,
      "loss": 0.2055,
      "rewards/chosen": 0.8088045120239258,
      "rewards/margins": 7.324716091156006,
      "rewards/rejected": -6.515912055969238,
      "step": 175
    },
    {
      "epoch": 0.12,
      "grad_norm": 28.048669815063477,
      "kl": 0.0,
      "learning_rate": 4.918207681365576e-07,
      "logps/chosen": -278.0254821777344,
      "logps/rejected": -392.7597351074219,
      "loss": 0.2087,
      "rewards/chosen": 0.7063338756561279,
      "rewards/margins": 7.6127119064331055,
      "rewards/rejected": -6.906378269195557,
      "step": 180
    },
    {
      "epoch": 0.12,
      "grad_norm": 29.8698673248291,
      "kl": 0.0,
      "learning_rate": 4.900426742532006e-07,
      "logps/chosen": -272.4722900390625,
      "logps/rejected": -392.2569580078125,
      "loss": 0.2483,
      "rewards/chosen": 0.6673704385757446,
      "rewards/margins": 6.765078067779541,
      "rewards/rejected": -6.097708225250244,
      "step": 185
    },
    {
      "epoch": 0.12,
      "grad_norm": 27.31839370727539,
      "kl": 0.0,
      "learning_rate": 4.882645803698435e-07,
      "logps/chosen": -295.7112731933594,
      "logps/rejected": -434.19317626953125,
      "loss": 0.2083,
      "rewards/chosen": 0.6688440442085266,
      "rewards/margins": 7.306863307952881,
      "rewards/rejected": -6.638019561767578,
      "step": 190
    },
    {
      "epoch": 0.12,
      "grad_norm": 35.319053649902344,
      "kl": 0.0,
      "learning_rate": 4.864864864864865e-07,
      "logps/chosen": -283.3681945800781,
      "logps/rejected": -392.4473571777344,
      "loss": 0.2282,
      "rewards/chosen": 0.6834143400192261,
      "rewards/margins": 7.1807074546813965,
      "rewards/rejected": -6.497292995452881,
      "step": 195
    },
    {
      "epoch": 0.13,
      "grad_norm": 28.466360092163086,
      "kl": 0.0,
      "learning_rate": 4.847083926031294e-07,
      "logps/chosen": -301.3221130371094,
      "logps/rejected": -402.22845458984375,
      "loss": 0.2128,
      "rewards/chosen": 0.8221393823623657,
      "rewards/margins": 7.278465270996094,
      "rewards/rejected": -6.456326484680176,
      "step": 200
    },
    {
      "epoch": 0.13,
      "grad_norm": 27.28252601623535,
      "kl": 0.0,
      "learning_rate": 4.829302987197724e-07,
      "logps/chosen": -306.77642822265625,
      "logps/rejected": -413.36346435546875,
      "loss": 0.2073,
      "rewards/chosen": 0.6411277651786804,
      "rewards/margins": 7.287237644195557,
      "rewards/rejected": -6.646109580993652,
      "step": 205
    },
    {
      "epoch": 0.13,
      "grad_norm": 32.64134979248047,
      "kl": 0.0,
      "learning_rate": 4.811522048364154e-07,
      "logps/chosen": -291.44818115234375,
      "logps/rejected": -427.9637756347656,
      "loss": 0.2164,
      "rewards/chosen": 0.8150373697280884,
      "rewards/margins": 7.651273250579834,
      "rewards/rejected": -6.836236476898193,
      "step": 210
    },
    {
      "epoch": 0.14,
      "grad_norm": 23.997915267944336,
      "kl": 0.0,
      "learning_rate": 4.793741109530583e-07,
      "logps/chosen": -260.71527099609375,
      "logps/rejected": -391.9830627441406,
      "loss": 0.2175,
      "rewards/chosen": 0.8054065704345703,
      "rewards/margins": 7.5576653480529785,
      "rewards/rejected": -6.752259254455566,
      "step": 215
    },
    {
      "epoch": 0.14,
      "grad_norm": 29.320816040039062,
      "kl": 0.0,
      "learning_rate": 4.775960170697012e-07,
      "logps/chosen": -268.3311462402344,
      "logps/rejected": -404.9643249511719,
      "loss": 0.2104,
      "rewards/chosen": 0.7653949856758118,
      "rewards/margins": 7.521485805511475,
      "rewards/rejected": -6.756091117858887,
      "step": 220
    },
    {
      "epoch": 0.14,
      "grad_norm": 27.857921600341797,
      "kl": 0.0,
      "learning_rate": 4.7581792318634425e-07,
      "logps/chosen": -290.99407958984375,
      "logps/rejected": -405.1004943847656,
      "loss": 0.2164,
      "rewards/chosen": 0.7016764879226685,
      "rewards/margins": 7.881253242492676,
      "rewards/rejected": -7.179576873779297,
      "step": 225
    },
    {
      "epoch": 0.15,
      "grad_norm": 26.333301544189453,
      "kl": 0.0,
      "learning_rate": 4.7403982930298717e-07,
      "logps/chosen": -246.4652557373047,
      "logps/rejected": -408.4164123535156,
      "loss": 0.2048,
      "rewards/chosen": 0.7065305709838867,
      "rewards/margins": 7.9227142333984375,
      "rewards/rejected": -7.216183662414551,
      "step": 230
    },
    {
      "epoch": 0.15,
      "grad_norm": 29.780120849609375,
      "kl": 0.0,
      "learning_rate": 4.7226173541963014e-07,
      "logps/chosen": -300.751953125,
      "logps/rejected": -409.5931091308594,
      "loss": 0.2022,
      "rewards/chosen": 0.8955439329147339,
      "rewards/margins": 8.069990158081055,
      "rewards/rejected": -7.174446105957031,
      "step": 235
    },
    {
      "epoch": 0.15,
      "grad_norm": 26.441162109375,
      "kl": 0.0,
      "learning_rate": 4.7048364153627306e-07,
      "logps/chosen": -298.4041442871094,
      "logps/rejected": -396.9103088378906,
      "loss": 0.2223,
      "rewards/chosen": 0.9372671842575073,
      "rewards/margins": 7.630146026611328,
      "rewards/rejected": -6.692878723144531,
      "step": 240
    },
    {
      "epoch": 0.16,
      "grad_norm": 27.794498443603516,
      "kl": 0.0,
      "learning_rate": 4.6870554765291604e-07,
      "logps/chosen": -304.92340087890625,
      "logps/rejected": -444.83526611328125,
      "loss": 0.1866,
      "rewards/chosen": 0.7705713510513306,
      "rewards/margins": 8.343452453613281,
      "rewards/rejected": -7.57288122177124,
      "step": 245
    },
    {
      "epoch": 0.16,
      "grad_norm": 27.902463912963867,
      "kl": 0.0,
      "learning_rate": 4.66927453769559e-07,
      "logps/chosen": -277.5145263671875,
      "logps/rejected": -426.4693908691406,
      "loss": 0.2184,
      "rewards/chosen": 0.719990074634552,
      "rewards/margins": 8.114697456359863,
      "rewards/rejected": -7.394706726074219,
      "step": 250
    },
    {
      "epoch": 0.16,
      "grad_norm": 33.13041305541992,
      "kl": 0.0,
      "learning_rate": 4.65149359886202e-07,
      "logps/chosen": -333.77032470703125,
      "logps/rejected": -404.6194152832031,
      "loss": 0.2038,
      "rewards/chosen": 0.7432358860969543,
      "rewards/margins": 8.134955406188965,
      "rewards/rejected": -7.391718864440918,
      "step": 255
    },
    {
      "epoch": 0.17,
      "grad_norm": 29.542213439941406,
      "kl": 0.0,
      "learning_rate": 4.633712660028449e-07,
      "logps/chosen": -286.2034606933594,
      "logps/rejected": -383.937744140625,
      "loss": 0.2245,
      "rewards/chosen": 0.6780925989151001,
      "rewards/margins": 6.890599727630615,
      "rewards/rejected": -6.212507724761963,
      "step": 260
    },
    {
      "epoch": 0.17,
      "grad_norm": 25.915569305419922,
      "kl": 0.0,
      "learning_rate": 4.615931721194879e-07,
      "logps/chosen": -287.2189025878906,
      "logps/rejected": -431.18115234375,
      "loss": 0.2253,
      "rewards/chosen": 0.7109718918800354,
      "rewards/margins": 7.668128967285156,
      "rewards/rejected": -6.957158088684082,
      "step": 265
    },
    {
      "epoch": 0.17,
      "grad_norm": 28.604249954223633,
      "kl": 0.0,
      "learning_rate": 4.5981507823613085e-07,
      "logps/chosen": -249.9962158203125,
      "logps/rejected": -427.6434631347656,
      "loss": 0.1922,
      "rewards/chosen": 0.9254335165023804,
      "rewards/margins": 8.525291442871094,
      "rewards/rejected": -7.599857330322266,
      "step": 270
    },
    {
      "epoch": 0.18,
      "grad_norm": 32.13933563232422,
      "kl": 0.0,
      "learning_rate": 4.580369843527738e-07,
      "logps/chosen": -250.48397827148438,
      "logps/rejected": -437.86541748046875,
      "loss": 0.2104,
      "rewards/chosen": 0.916563868522644,
      "rewards/margins": 8.427824020385742,
      "rewards/rejected": -7.511260986328125,
      "step": 275
    },
    {
      "epoch": 0.18,
      "grad_norm": 24.797935485839844,
      "kl": 0.0,
      "learning_rate": 4.562588904694168e-07,
      "logps/chosen": -289.09368896484375,
      "logps/rejected": -382.2262268066406,
      "loss": 0.2292,
      "rewards/chosen": 0.8693010210990906,
      "rewards/margins": 8.239991188049316,
      "rewards/rejected": -7.37069034576416,
      "step": 280
    },
    {
      "epoch": 0.18,
      "grad_norm": 29.941953659057617,
      "kl": 0.0,
      "learning_rate": 4.544807965860597e-07,
      "logps/chosen": -286.82781982421875,
      "logps/rejected": -399.9955139160156,
      "loss": 0.2097,
      "rewards/chosen": 1.1082446575164795,
      "rewards/margins": 8.612449645996094,
      "rewards/rejected": -7.504205226898193,
      "step": 285
    },
    {
      "epoch": 0.19,
      "grad_norm": 28.574798583984375,
      "kl": 0.0,
      "learning_rate": 4.5270270270270264e-07,
      "logps/chosen": -296.76751708984375,
      "logps/rejected": -426.483154296875,
      "loss": 0.201,
      "rewards/chosen": 1.1846811771392822,
      "rewards/margins": 8.457054138183594,
      "rewards/rejected": -7.272373199462891,
      "step": 290
    },
    {
      "epoch": 0.19,
      "grad_norm": 24.285078048706055,
      "kl": 0.0,
      "learning_rate": 4.509246088193456e-07,
      "logps/chosen": -253.2152099609375,
      "logps/rejected": -407.7524719238281,
      "loss": 0.2124,
      "rewards/chosen": 0.8104816675186157,
      "rewards/margins": 8.143354415893555,
      "rewards/rejected": -7.332873344421387,
      "step": 295
    },
    {
      "epoch": 0.19,
      "grad_norm": 28.82489585876465,
      "kl": 0.0,
      "learning_rate": 4.491465149359886e-07,
      "logps/chosen": -282.62725830078125,
      "logps/rejected": -399.4122009277344,
      "loss": 0.2058,
      "rewards/chosen": 0.829262375831604,
      "rewards/margins": 8.509421348571777,
      "rewards/rejected": -7.680159568786621,
      "step": 300
    },
    {
      "epoch": 0.2,
      "grad_norm": 26.383644104003906,
      "kl": 0.0,
      "learning_rate": 4.4736842105263156e-07,
      "logps/chosen": -253.8015899658203,
      "logps/rejected": -423.59454345703125,
      "loss": 0.1916,
      "rewards/chosen": 0.9279552698135376,
      "rewards/margins": 8.754355430603027,
      "rewards/rejected": -7.8264007568359375,
      "step": 305
    },
    {
      "epoch": 0.2,
      "grad_norm": 30.41068458557129,
      "kl": 0.0,
      "learning_rate": 4.4559032716927454e-07,
      "logps/chosen": -306.1068420410156,
      "logps/rejected": -404.15777587890625,
      "loss": 0.1763,
      "rewards/chosen": 1.2404173612594604,
      "rewards/margins": 8.863131523132324,
      "rewards/rejected": -7.622714042663574,
      "step": 310
    },
    {
      "epoch": 0.2,
      "grad_norm": 27.128496170043945,
      "kl": 0.0,
      "learning_rate": 4.438122332859175e-07,
      "logps/chosen": -288.37420654296875,
      "logps/rejected": -434.837158203125,
      "loss": 0.2103,
      "rewards/chosen": 0.9712983965873718,
      "rewards/margins": 8.523191452026367,
      "rewards/rejected": -7.551893711090088,
      "step": 315
    },
    {
      "epoch": 0.2,
      "grad_norm": 24.96108627319336,
      "kl": 0.0,
      "learning_rate": 4.420341394025605e-07,
      "logps/chosen": -225.9550018310547,
      "logps/rejected": -397.37725830078125,
      "loss": 0.1946,
      "rewards/chosen": 0.9277496337890625,
      "rewards/margins": 8.359576225280762,
      "rewards/rejected": -7.431826591491699,
      "step": 320
    },
    {
      "epoch": 0.21,
      "grad_norm": 27.4498348236084,
      "kl": 0.0,
      "learning_rate": 4.4025604551920335e-07,
      "logps/chosen": -326.1064453125,
      "logps/rejected": -457.33917236328125,
      "loss": 0.1974,
      "rewards/chosen": 0.9113430976867676,
      "rewards/margins": 9.018231391906738,
      "rewards/rejected": -8.106887817382812,
      "step": 325
    },
    {
      "epoch": 0.21,
      "grad_norm": 26.263656616210938,
      "kl": 0.0,
      "learning_rate": 4.384779516358463e-07,
      "logps/chosen": -264.4283752441406,
      "logps/rejected": -424.5361328125,
      "loss": 0.2007,
      "rewards/chosen": 0.9189130663871765,
      "rewards/margins": 8.478607177734375,
      "rewards/rejected": -7.559694766998291,
      "step": 330
    },
    {
      "epoch": 0.21,
      "grad_norm": 32.865238189697266,
      "kl": 0.0,
      "learning_rate": 4.366998577524893e-07,
      "logps/chosen": -321.8217468261719,
      "logps/rejected": -422.74346923828125,
      "loss": 0.1828,
      "rewards/chosen": 1.213265299797058,
      "rewards/margins": 9.324636459350586,
      "rewards/rejected": -8.111371994018555,
      "step": 335
    },
    {
      "epoch": 0.22,
      "grad_norm": 25.276630401611328,
      "kl": 0.0,
      "learning_rate": 4.3492176386913227e-07,
      "logps/chosen": -286.50762939453125,
      "logps/rejected": -416.09246826171875,
      "loss": 0.1918,
      "rewards/chosen": 1.121449589729309,
      "rewards/margins": 8.465108871459961,
      "rewards/rejected": -7.343659400939941,
      "step": 340
    },
    {
      "epoch": 0.22,
      "grad_norm": 33.83981704711914,
      "kl": 0.0,
      "learning_rate": 4.3314366998577524e-07,
      "logps/chosen": -310.34197998046875,
      "logps/rejected": -378.6121826171875,
      "loss": 0.2018,
      "rewards/chosen": 1.2548576593399048,
      "rewards/margins": 8.125253677368164,
      "rewards/rejected": -6.870396614074707,
      "step": 345
    },
    {
      "epoch": 0.22,
      "grad_norm": 33.48633575439453,
      "kl": 0.0,
      "learning_rate": 4.313655761024182e-07,
      "logps/chosen": -281.07489013671875,
      "logps/rejected": -429.463134765625,
      "loss": 0.193,
      "rewards/chosen": 1.1550464630126953,
      "rewards/margins": 9.02354907989502,
      "rewards/rejected": -7.868502616882324,
      "step": 350
    },
    {
      "epoch": 0.23,
      "grad_norm": 24.945789337158203,
      "kl": 0.0,
      "learning_rate": 4.2958748221906114e-07,
      "logps/chosen": -272.5113830566406,
      "logps/rejected": -378.4676513671875,
      "loss": 0.1983,
      "rewards/chosen": 1.260040044784546,
      "rewards/margins": 8.773920059204102,
      "rewards/rejected": -7.513879299163818,
      "step": 355
    },
    {
      "epoch": 0.23,
      "grad_norm": 27.782686233520508,
      "kl": 0.0,
      "learning_rate": 4.278093883357041e-07,
      "logps/chosen": -265.1444396972656,
      "logps/rejected": -415.40264892578125,
      "loss": 0.1671,
      "rewards/chosen": 1.351818323135376,
      "rewards/margins": 9.423505783081055,
      "rewards/rejected": -8.071688652038574,
      "step": 360
    },
    {
      "epoch": 0.23,
      "grad_norm": 25.29680824279785,
      "kl": 0.0,
      "learning_rate": 4.260312944523471e-07,
      "logps/chosen": -320.04193115234375,
      "logps/rejected": -462.45574951171875,
      "loss": 0.1719,
      "rewards/chosen": 1.2879645824432373,
      "rewards/margins": 9.490580558776855,
      "rewards/rejected": -8.202616691589355,
      "step": 365
    },
    {
      "epoch": 0.24,
      "grad_norm": 24.49195671081543,
      "kl": 0.0,
      "learning_rate": 4.2425320056899e-07,
      "logps/chosen": -289.80413818359375,
      "logps/rejected": -404.28094482421875,
      "loss": 0.1993,
      "rewards/chosen": 1.0154571533203125,
      "rewards/margins": 8.819131851196289,
      "rewards/rejected": -7.803675174713135,
      "step": 370
    },
    {
      "epoch": 0.24,
      "grad_norm": 26.57324981689453,
      "kl": 0.0,
      "learning_rate": 4.22475106685633e-07,
      "logps/chosen": -249.9967041015625,
      "logps/rejected": -436.7953186035156,
      "loss": 0.1902,
      "rewards/chosen": 1.0482652187347412,
      "rewards/margins": 9.115781784057617,
      "rewards/rejected": -8.067517280578613,
      "step": 375
    },
    {
      "epoch": 0.24,
      "grad_norm": 28.738710403442383,
      "kl": 0.0,
      "learning_rate": 4.2069701280227595e-07,
      "logps/chosen": -241.308349609375,
      "logps/rejected": -425.63970947265625,
      "loss": 0.2145,
      "rewards/chosen": 0.9155824780464172,
      "rewards/margins": 8.491414070129395,
      "rewards/rejected": -7.575830936431885,
      "step": 380
    },
    {
      "epoch": 0.25,
      "grad_norm": 26.93291664123535,
      "kl": 0.0,
      "learning_rate": 4.189189189189189e-07,
      "logps/chosen": -298.3195495605469,
      "logps/rejected": -429.3912048339844,
      "loss": 0.1832,
      "rewards/chosen": 1.0986849069595337,
      "rewards/margins": 9.5584716796875,
      "rewards/rejected": -8.459787368774414,
      "step": 385
    },
    {
      "epoch": 0.25,
      "grad_norm": 28.49407386779785,
      "kl": 0.0,
      "learning_rate": 4.1714082503556185e-07,
      "logps/chosen": -289.1352844238281,
      "logps/rejected": -421.0469665527344,
      "loss": 0.1782,
      "rewards/chosen": 0.9938220977783203,
      "rewards/margins": 9.06061840057373,
      "rewards/rejected": -8.066795349121094,
      "step": 390
    },
    {
      "epoch": 0.25,
      "grad_norm": 23.859844207763672,
      "kl": 0.0,
      "learning_rate": 4.153627311522048e-07,
      "logps/chosen": -259.1231689453125,
      "logps/rejected": -452.96746826171875,
      "loss": 0.1904,
      "rewards/chosen": 0.8238496780395508,
      "rewards/margins": 9.638608932495117,
      "rewards/rejected": -8.814759254455566,
      "step": 395
    },
    {
      "epoch": 0.26,
      "grad_norm": 32.167110443115234,
      "kl": 0.0,
      "learning_rate": 4.135846372688478e-07,
      "logps/chosen": -295.9509582519531,
      "logps/rejected": -448.93377685546875,
      "loss": 0.1753,
      "rewards/chosen": 0.963021457195282,
      "rewards/margins": 9.281633377075195,
      "rewards/rejected": -8.318610191345215,
      "step": 400
    },
    {
      "epoch": 0.26,
      "grad_norm": 27.581504821777344,
      "kl": 0.0,
      "learning_rate": 4.1180654338549077e-07,
      "logps/chosen": -281.5043029785156,
      "logps/rejected": -388.4980163574219,
      "loss": 0.2066,
      "rewards/chosen": 1.0012767314910889,
      "rewards/margins": 8.807787895202637,
      "rewards/rejected": -7.806510925292969,
      "step": 405
    },
    {
      "epoch": 0.26,
      "grad_norm": 29.895170211791992,
      "kl": 0.0,
      "learning_rate": 4.100284495021337e-07,
      "logps/chosen": -287.0378723144531,
      "logps/rejected": -427.29864501953125,
      "loss": 0.2007,
      "rewards/chosen": 0.8726502656936646,
      "rewards/margins": 9.106569290161133,
      "rewards/rejected": -8.233919143676758,
      "step": 410
    },
    {
      "epoch": 0.27,
      "grad_norm": 24.92208480834961,
      "kl": 0.0,
      "learning_rate": 4.082503556187766e-07,
      "logps/chosen": -268.7417297363281,
      "logps/rejected": -411.2549743652344,
      "loss": 0.1975,
      "rewards/chosen": 0.902958869934082,
      "rewards/margins": 8.691009521484375,
      "rewards/rejected": -7.788050174713135,
      "step": 415
    },
    {
      "epoch": 0.27,
      "grad_norm": 30.231552124023438,
      "kl": 0.0,
      "learning_rate": 4.064722617354196e-07,
      "logps/chosen": -286.7518310546875,
      "logps/rejected": -451.4786071777344,
      "loss": 0.1867,
      "rewards/chosen": 1.2976207733154297,
      "rewards/margins": 9.737428665161133,
      "rewards/rejected": -8.43980884552002,
      "step": 420
    },
    {
      "epoch": 0.27,
      "grad_norm": 24.8756046295166,
      "kl": 0.0,
      "learning_rate": 4.0469416785206256e-07,
      "logps/chosen": -282.55706787109375,
      "logps/rejected": -433.0348205566406,
      "loss": 0.1846,
      "rewards/chosen": 1.047466516494751,
      "rewards/margins": 9.786076545715332,
      "rewards/rejected": -8.738609313964844,
      "step": 425
    },
    {
      "epoch": 0.28,
      "grad_norm": 29.7546329498291,
      "kl": 0.0,
      "learning_rate": 4.0291607396870553e-07,
      "logps/chosen": -285.54852294921875,
      "logps/rejected": -450.5826110839844,
      "loss": 0.1708,
      "rewards/chosen": 1.2604820728302002,
      "rewards/margins": 9.911845207214355,
      "rewards/rejected": -8.651362419128418,
      "step": 430
    },
    {
      "epoch": 0.28,
      "grad_norm": 26.89280891418457,
      "kl": 0.0,
      "learning_rate": 4.011379800853485e-07,
      "logps/chosen": -274.69195556640625,
      "logps/rejected": -415.19989013671875,
      "loss": 0.1939,
      "rewards/chosen": 1.1895002126693726,
      "rewards/margins": 9.175015449523926,
      "rewards/rejected": -7.985515594482422,
      "step": 435
    },
    {
      "epoch": 0.28,
      "grad_norm": 49.50102996826172,
      "kl": 0.0,
      "learning_rate": 3.993598862019915e-07,
      "logps/chosen": -257.03985595703125,
      "logps/rejected": -437.6024475097656,
      "loss": 0.179,
      "rewards/chosen": 1.1160862445831299,
      "rewards/margins": 10.291946411132812,
      "rewards/rejected": -9.175861358642578,
      "step": 440
    },
    {
      "epoch": 0.28,
      "grad_norm": 23.507465362548828,
      "kl": 0.0,
      "learning_rate": 3.975817923186344e-07,
      "logps/chosen": -244.9132537841797,
      "logps/rejected": -417.92010498046875,
      "loss": 0.1943,
      "rewards/chosen": 0.9916771054267883,
      "rewards/margins": 8.110875129699707,
      "rewards/rejected": -7.119197845458984,
      "step": 445
    },
    {
      "epoch": 0.29,
      "grad_norm": 23.709091186523438,
      "kl": 0.0,
      "learning_rate": 3.9580369843527737e-07,
      "logps/chosen": -266.93243408203125,
      "logps/rejected": -474.86004638671875,
      "loss": 0.1754,
      "rewards/chosen": 1.332706093788147,
      "rewards/margins": 9.872304916381836,
      "rewards/rejected": -8.53959846496582,
      "step": 450
    },
    {
      "epoch": 0.29,
      "grad_norm": 28.132789611816406,
      "kl": 0.0,
      "learning_rate": 3.940256045519203e-07,
      "logps/chosen": -269.0207824707031,
      "logps/rejected": -441.16259765625,
      "loss": 0.1764,
      "rewards/chosen": 1.136040449142456,
      "rewards/margins": 9.543194770812988,
      "rewards/rejected": -8.407154083251953,
      "step": 455
    },
    {
      "epoch": 0.29,
      "grad_norm": 27.44135284423828,
      "kl": 0.0,
      "learning_rate": 3.9224751066856327e-07,
      "logps/chosen": -261.8495178222656,
      "logps/rejected": -416.13372802734375,
      "loss": 0.2019,
      "rewards/chosen": 1.0264383554458618,
      "rewards/margins": 9.626348495483398,
      "rewards/rejected": -8.599910736083984,
      "step": 460
    },
    {
      "epoch": 0.3,
      "grad_norm": 34.613067626953125,
      "kl": 0.0,
      "learning_rate": 3.9046941678520624e-07,
      "logps/chosen": -305.896240234375,
      "logps/rejected": -436.7113342285156,
      "loss": 0.2045,
      "rewards/chosen": 1.227920651435852,
      "rewards/margins": 9.572298049926758,
      "rewards/rejected": -8.344377517700195,
      "step": 465
    },
    {
      "epoch": 0.3,
      "grad_norm": 31.94422721862793,
      "kl": 0.0,
      "learning_rate": 3.886913229018492e-07,
      "logps/chosen": -275.5303955078125,
      "logps/rejected": -421.15020751953125,
      "loss": 0.2019,
      "rewards/chosen": 1.0201870203018188,
      "rewards/margins": 9.202049255371094,
      "rewards/rejected": -8.181862831115723,
      "step": 470
    },
    {
      "epoch": 0.3,
      "grad_norm": 24.941471099853516,
      "kl": 0.0,
      "learning_rate": 3.8691322901849213e-07,
      "logps/chosen": -309.7427978515625,
      "logps/rejected": -456.80987548828125,
      "loss": 0.1867,
      "rewards/chosen": 1.4211246967315674,
      "rewards/margins": 9.695344924926758,
      "rewards/rejected": -8.274219512939453,
      "step": 475
    },
    {
      "epoch": 0.31,
      "grad_norm": 25.9896240234375,
      "kl": 0.0,
      "learning_rate": 3.851351351351351e-07,
      "logps/chosen": -285.4349365234375,
      "logps/rejected": -400.69219970703125,
      "loss": 0.1817,
      "rewards/chosen": 1.505924105644226,
      "rewards/margins": 8.47026538848877,
      "rewards/rejected": -6.964341640472412,
      "step": 480
    },
    {
      "epoch": 0.31,
      "grad_norm": 29.482158660888672,
      "kl": 0.0,
      "learning_rate": 3.833570412517781e-07,
      "logps/chosen": -262.8547058105469,
      "logps/rejected": -404.0810546875,
      "loss": 0.1838,
      "rewards/chosen": 1.2003206014633179,
      "rewards/margins": 8.89756965637207,
      "rewards/rejected": -7.697249412536621,
      "step": 485
    },
    {
      "epoch": 0.31,
      "grad_norm": 24.927303314208984,
      "kl": 0.0,
      "learning_rate": 3.8157894736842105e-07,
      "logps/chosen": -279.3279724121094,
      "logps/rejected": -426.81646728515625,
      "loss": 0.1816,
      "rewards/chosen": 1.2700178623199463,
      "rewards/margins": 9.352417945861816,
      "rewards/rejected": -8.082399368286133,
      "step": 490
    },
    {
      "epoch": 0.32,
      "grad_norm": 19.47848892211914,
      "kl": 0.0,
      "learning_rate": 3.7980085348506403e-07,
      "logps/chosen": -254.8472137451172,
      "logps/rejected": -410.69219970703125,
      "loss": 0.1617,
      "rewards/chosen": 1.251443862915039,
      "rewards/margins": 9.37706184387207,
      "rewards/rejected": -8.125617980957031,
      "step": 495
    },
    {
      "epoch": 0.32,
      "grad_norm": 27.02752685546875,
      "kl": 0.0,
      "learning_rate": 3.7802275960170695e-07,
      "logps/chosen": -326.91485595703125,
      "logps/rejected": -427.81121826171875,
      "loss": 0.1869,
      "rewards/chosen": 1.2399226427078247,
      "rewards/margins": 9.168660163879395,
      "rewards/rejected": -7.928736686706543,
      "step": 500
    },
    {
      "epoch": 0.32,
      "grad_norm": 23.25274658203125,
      "kl": 0.0,
      "learning_rate": 3.7624466571834987e-07,
      "logps/chosen": -263.9060363769531,
      "logps/rejected": -486.6805114746094,
      "loss": 0.1867,
      "rewards/chosen": 1.059422254562378,
      "rewards/margins": 9.522220611572266,
      "rewards/rejected": -8.462798118591309,
      "step": 505
    },
    {
      "epoch": 0.33,
      "grad_norm": 28.608474731445312,
      "kl": 0.0,
      "learning_rate": 3.7446657183499284e-07,
      "logps/chosen": -294.48590087890625,
      "logps/rejected": -468.9127502441406,
      "loss": 0.1785,
      "rewards/chosen": 1.3862662315368652,
      "rewards/margins": 10.08003044128418,
      "rewards/rejected": -8.693764686584473,
      "step": 510
    },
    {
      "epoch": 0.33,
      "grad_norm": 28.52597999572754,
      "kl": 0.0,
      "learning_rate": 3.726884779516358e-07,
      "logps/chosen": -303.76116943359375,
      "logps/rejected": -385.89471435546875,
      "loss": 0.1849,
      "rewards/chosen": 0.8810665011405945,
      "rewards/margins": 8.036872863769531,
      "rewards/rejected": -7.155806064605713,
      "step": 515
    },
    {
      "epoch": 0.33,
      "grad_norm": 26.567203521728516,
      "kl": 0.0,
      "learning_rate": 3.709103840682788e-07,
      "logps/chosen": -300.5899963378906,
      "logps/rejected": -455.53887939453125,
      "loss": 0.1752,
      "rewards/chosen": 1.6550683975219727,
      "rewards/margins": 9.88080883026123,
      "rewards/rejected": -8.225740432739258,
      "step": 520
    },
    {
      "epoch": 0.34,
      "grad_norm": 26.40464973449707,
      "kl": 0.0,
      "learning_rate": 3.6913229018492176e-07,
      "logps/chosen": -291.1101989746094,
      "logps/rejected": -429.8409118652344,
      "loss": 0.1738,
      "rewards/chosen": 1.6012027263641357,
      "rewards/margins": 10.346236228942871,
      "rewards/rejected": -8.745033264160156,
      "step": 525
    },
    {
      "epoch": 0.34,
      "grad_norm": 25.405059814453125,
      "kl": 0.0,
      "learning_rate": 3.6735419630156474e-07,
      "logps/chosen": -277.75,
      "logps/rejected": -444.23687744140625,
      "loss": 0.1632,
      "rewards/chosen": 1.4320881366729736,
      "rewards/margins": 10.243904113769531,
      "rewards/rejected": -8.811816215515137,
      "step": 530
    },
    {
      "epoch": 0.34,
      "grad_norm": 26.102277755737305,
      "kl": 0.0,
      "learning_rate": 3.655761024182077e-07,
      "logps/chosen": -275.7671813964844,
      "logps/rejected": -451.432861328125,
      "loss": 0.1818,
      "rewards/chosen": 1.5062631368637085,
      "rewards/margins": 9.949407577514648,
      "rewards/rejected": -8.443144798278809,
      "step": 535
    },
    {
      "epoch": 0.35,
      "grad_norm": 27.81848907470703,
      "kl": 0.0,
      "learning_rate": 3.637980085348506e-07,
      "logps/chosen": -285.43463134765625,
      "logps/rejected": -456.70257568359375,
      "loss": 0.1667,
      "rewards/chosen": 1.1070717573165894,
      "rewards/margins": 10.036450386047363,
      "rewards/rejected": -8.9293794631958,
      "step": 540
    },
    {
      "epoch": 0.35,
      "grad_norm": 26.733715057373047,
      "kl": 0.0,
      "learning_rate": 3.6201991465149355e-07,
      "logps/chosen": -275.365478515625,
      "logps/rejected": -447.39874267578125,
      "loss": 0.1584,
      "rewards/chosen": 1.669955849647522,
      "rewards/margins": 9.940733909606934,
      "rewards/rejected": -8.27077865600586,
      "step": 545
    },
    {
      "epoch": 0.35,
      "grad_norm": 28.002460479736328,
      "kl": 0.0,
      "learning_rate": 3.602418207681365e-07,
      "logps/chosen": -275.7063903808594,
      "logps/rejected": -432.69097900390625,
      "loss": 0.1961,
      "rewards/chosen": 1.1493828296661377,
      "rewards/margins": 9.39341926574707,
      "rewards/rejected": -8.244035720825195,
      "step": 550
    },
    {
      "epoch": 0.36,
      "grad_norm": 25.290157318115234,
      "kl": 0.0,
      "learning_rate": 3.584637268847795e-07,
      "logps/chosen": -276.50872802734375,
      "logps/rejected": -427.5359802246094,
      "loss": 0.1825,
      "rewards/chosen": 1.6107838153839111,
      "rewards/margins": 10.058174133300781,
      "rewards/rejected": -8.447389602661133,
      "step": 555
    },
    {
      "epoch": 0.36,
      "grad_norm": 27.16476058959961,
      "kl": 0.0,
      "learning_rate": 3.5668563300142247e-07,
      "logps/chosen": -259.827880859375,
      "logps/rejected": -409.2623596191406,
      "loss": 0.1808,
      "rewards/chosen": 1.3736449480056763,
      "rewards/margins": 9.484289169311523,
      "rewards/rejected": -8.110644340515137,
      "step": 560
    },
    {
      "epoch": 0.36,
      "grad_norm": 28.099790573120117,
      "kl": 0.0,
      "learning_rate": 3.5490753911806545e-07,
      "logps/chosen": -279.4620666503906,
      "logps/rejected": -404.58563232421875,
      "loss": 0.185,
      "rewards/chosen": 1.319478988647461,
      "rewards/margins": 9.3961181640625,
      "rewards/rejected": -8.076639175415039,
      "step": 565
    },
    {
      "epoch": 0.36,
      "grad_norm": 28.673524856567383,
      "kl": 0.0,
      "learning_rate": 3.5312944523470837e-07,
      "logps/chosen": -283.595703125,
      "logps/rejected": -403.426025390625,
      "loss": 0.1838,
      "rewards/chosen": 1.2756729125976562,
      "rewards/margins": 9.318367958068848,
      "rewards/rejected": -8.042695999145508,
      "step": 570
    },
    {
      "epoch": 0.37,
      "grad_norm": 24.276857376098633,
      "kl": 0.0,
      "learning_rate": 3.5135135135135134e-07,
      "logps/chosen": -278.02337646484375,
      "logps/rejected": -390.92987060546875,
      "loss": 0.189,
      "rewards/chosen": 1.1147047281265259,
      "rewards/margins": 9.350360870361328,
      "rewards/rejected": -8.235654830932617,
      "step": 575
    },
    {
      "epoch": 0.37,
      "grad_norm": 24.185720443725586,
      "kl": 0.0,
      "learning_rate": 3.495732574679943e-07,
      "logps/chosen": -317.0126953125,
      "logps/rejected": -441.64324951171875,
      "loss": 0.1709,
      "rewards/chosen": 1.2199360132217407,
      "rewards/margins": 10.402612686157227,
      "rewards/rejected": -9.182676315307617,
      "step": 580
    },
    {
      "epoch": 0.37,
      "grad_norm": 21.31283187866211,
      "kl": 0.0,
      "learning_rate": 3.4779516358463724e-07,
      "logps/chosen": -299.08355712890625,
      "logps/rejected": -408.64312744140625,
      "loss": 0.1669,
      "rewards/chosen": 1.33742356300354,
      "rewards/margins": 9.711603164672852,
      "rewards/rejected": -8.374178886413574,
      "step": 585
    },
    {
      "epoch": 0.38,
      "grad_norm": 27.55930519104004,
      "kl": 0.0,
      "learning_rate": 3.460170697012802e-07,
      "logps/chosen": -358.379150390625,
      "logps/rejected": -390.5495910644531,
      "loss": 0.1711,
      "rewards/chosen": 1.5247728824615479,
      "rewards/margins": 10.199918746948242,
      "rewards/rejected": -8.675146102905273,
      "step": 590
    },
    {
      "epoch": 0.38,
      "grad_norm": 22.836402893066406,
      "kl": 0.0,
      "learning_rate": 3.442389758179232e-07,
      "logps/chosen": -265.90155029296875,
      "logps/rejected": -407.1119079589844,
      "loss": 0.1813,
      "rewards/chosen": 1.2516599893569946,
      "rewards/margins": 9.78925895690918,
      "rewards/rejected": -8.53760051727295,
      "step": 595
    },
    {
      "epoch": 0.38,
      "grad_norm": 22.36507225036621,
      "kl": 0.0,
      "learning_rate": 3.424608819345661e-07,
      "logps/chosen": -272.3090515136719,
      "logps/rejected": -486.1949157714844,
      "loss": 0.1542,
      "rewards/chosen": 1.5906411409378052,
      "rewards/margins": 10.901646614074707,
      "rewards/rejected": -9.311005592346191,
      "step": 600
    },
    {
      "epoch": 0.39,
      "grad_norm": 25.3875732421875,
      "kl": 0.0,
      "learning_rate": 3.406827880512091e-07,
      "logps/chosen": -263.0488586425781,
      "logps/rejected": -443.8939514160156,
      "loss": 0.1639,
      "rewards/chosen": 1.4088127613067627,
      "rewards/margins": 10.464536666870117,
      "rewards/rejected": -9.05572509765625,
      "step": 605
    },
    {
      "epoch": 0.39,
      "grad_norm": 49.29353713989258,
      "kl": 0.0,
      "learning_rate": 3.3890469416785205e-07,
      "logps/chosen": -277.53131103515625,
      "logps/rejected": -402.04229736328125,
      "loss": 0.1994,
      "rewards/chosen": 1.2445075511932373,
      "rewards/margins": 9.684932708740234,
      "rewards/rejected": -8.440424919128418,
      "step": 610
    },
    {
      "epoch": 0.39,
      "grad_norm": 21.34065055847168,
      "kl": 0.0,
      "learning_rate": 3.37126600284495e-07,
      "logps/chosen": -231.4390106201172,
      "logps/rejected": -402.5743103027344,
      "loss": 0.1749,
      "rewards/chosen": 1.636435866355896,
      "rewards/margins": 9.378325462341309,
      "rewards/rejected": -7.741889953613281,
      "step": 615
    },
    {
      "epoch": 0.4,
      "grad_norm": 28.961084365844727,
      "kl": 0.0,
      "learning_rate": 3.35348506401138e-07,
      "logps/chosen": -255.66268920898438,
      "logps/rejected": -418.0624084472656,
      "loss": 0.1836,
      "rewards/chosen": 1.2639700174331665,
      "rewards/margins": 9.351852416992188,
      "rewards/rejected": -8.087881088256836,
      "step": 620
    },
    {
      "epoch": 0.4,
      "grad_norm": 28.00955581665039,
      "kl": 0.0,
      "learning_rate": 3.335704125177809e-07,
      "logps/chosen": -257.3089599609375,
      "logps/rejected": -375.06072998046875,
      "loss": 0.1711,
      "rewards/chosen": 1.5304003953933716,
      "rewards/margins": 9.457714080810547,
      "rewards/rejected": -7.927313804626465,
      "step": 625
    },
    {
      "epoch": 0.4,
      "grad_norm": 27.666671752929688,
      "kl": 0.0,
      "learning_rate": 3.3179231863442384e-07,
      "logps/chosen": -310.2152404785156,
      "logps/rejected": -440.0098571777344,
      "loss": 0.1814,
      "rewards/chosen": 1.2127114534378052,
      "rewards/margins": 10.335000991821289,
      "rewards/rejected": -9.122289657592773,
      "step": 630
    },
    {
      "epoch": 0.41,
      "grad_norm": 22.16933822631836,
      "kl": 0.0,
      "learning_rate": 3.300142247510668e-07,
      "logps/chosen": -260.0948181152344,
      "logps/rejected": -431.2792053222656,
      "loss": 0.1578,
      "rewards/chosen": 1.282593011856079,
      "rewards/margins": 9.947371482849121,
      "rewards/rejected": -8.664778709411621,
      "step": 635
    },
    {
      "epoch": 0.41,
      "grad_norm": 25.740550994873047,
      "kl": 0.0,
      "learning_rate": 3.282361308677098e-07,
      "logps/chosen": -293.82623291015625,
      "logps/rejected": -439.79083251953125,
      "loss": 0.1757,
      "rewards/chosen": 1.2234735488891602,
      "rewards/margins": 10.45723819732666,
      "rewards/rejected": -9.233762741088867,
      "step": 640
    },
    {
      "epoch": 0.41,
      "grad_norm": 23.11900520324707,
      "kl": 0.0,
      "learning_rate": 3.2645803698435276e-07,
      "logps/chosen": -284.302490234375,
      "logps/rejected": -444.50164794921875,
      "loss": 0.1794,
      "rewards/chosen": 1.5410137176513672,
      "rewards/margins": 10.157942771911621,
      "rewards/rejected": -8.616928100585938,
      "step": 645
    },
    {
      "epoch": 0.42,
      "grad_norm": 25.599075317382812,
      "kl": 0.0,
      "learning_rate": 3.2467994310099573e-07,
      "logps/chosen": -297.5779724121094,
      "logps/rejected": -416.24603271484375,
      "loss": 0.188,
      "rewards/chosen": 1.241750955581665,
      "rewards/margins": 10.071483612060547,
      "rewards/rejected": -8.829732894897461,
      "step": 650
    },
    {
      "epoch": 0.42,
      "grad_norm": 32.84114074707031,
      "kl": 0.0,
      "learning_rate": 3.229018492176387e-07,
      "logps/chosen": -295.51666259765625,
      "logps/rejected": -381.6005859375,
      "loss": 0.1772,
      "rewards/chosen": 1.4009755849838257,
      "rewards/margins": 10.052778244018555,
      "rewards/rejected": -8.651803016662598,
      "step": 655
    },
    {
      "epoch": 0.42,
      "grad_norm": 25.808273315429688,
      "kl": 0.0,
      "learning_rate": 3.211237553342817e-07,
      "logps/chosen": -252.7512969970703,
      "logps/rejected": -456.89727783203125,
      "loss": 0.1781,
      "rewards/chosen": 1.603581190109253,
      "rewards/margins": 11.127111434936523,
      "rewards/rejected": -9.523530960083008,
      "step": 660
    },
    {
      "epoch": 0.43,
      "grad_norm": 22.859973907470703,
      "kl": 0.0,
      "learning_rate": 3.193456614509246e-07,
      "logps/chosen": -257.5951232910156,
      "logps/rejected": -429.0784606933594,
      "loss": 0.1679,
      "rewards/chosen": 1.4137523174285889,
      "rewards/margins": 10.75071907043457,
      "rewards/rejected": -9.336965560913086,
      "step": 665
    },
    {
      "epoch": 0.43,
      "grad_norm": 20.03971290588379,
      "kl": 0.0,
      "learning_rate": 3.175675675675675e-07,
      "logps/chosen": -275.7456359863281,
      "logps/rejected": -432.35675048828125,
      "loss": 0.166,
      "rewards/chosen": 1.5532034635543823,
      "rewards/margins": 10.516412734985352,
      "rewards/rejected": -8.963208198547363,
      "step": 670
    },
    {
      "epoch": 0.43,
      "grad_norm": 25.613548278808594,
      "kl": 0.0,
      "learning_rate": 3.157894736842105e-07,
      "logps/chosen": -259.3094482421875,
      "logps/rejected": -428.52545166015625,
      "loss": 0.1715,
      "rewards/chosen": 1.5239208936691284,
      "rewards/margins": 10.248991012573242,
      "rewards/rejected": -8.72507095336914,
      "step": 675
    },
    {
      "epoch": 0.44,
      "grad_norm": 24.860313415527344,
      "kl": 0.0,
      "learning_rate": 3.1401137980085347e-07,
      "logps/chosen": -253.43408203125,
      "logps/rejected": -421.2649841308594,
      "loss": 0.1661,
      "rewards/chosen": 1.7669661045074463,
      "rewards/margins": 10.6715726852417,
      "rewards/rejected": -8.904605865478516,
      "step": 680
    },
    {
      "epoch": 0.44,
      "grad_norm": 20.709943771362305,
      "kl": 0.0,
      "learning_rate": 3.1223328591749644e-07,
      "logps/chosen": -309.0604248046875,
      "logps/rejected": -446.6376953125,
      "loss": 0.1677,
      "rewards/chosen": 1.6336666345596313,
      "rewards/margins": 10.273514747619629,
      "rewards/rejected": -8.639848709106445,
      "step": 685
    },
    {
      "epoch": 0.44,
      "grad_norm": 24.48786163330078,
      "kl": 0.0,
      "learning_rate": 3.104551920341394e-07,
      "logps/chosen": -258.40936279296875,
      "logps/rejected": -425.4024963378906,
      "loss": 0.1901,
      "rewards/chosen": 1.3902490139007568,
      "rewards/margins": 10.214468002319336,
      "rewards/rejected": -8.824217796325684,
      "step": 690
    },
    {
      "epoch": 0.44,
      "grad_norm": 23.349231719970703,
      "kl": 0.0,
      "learning_rate": 3.0867709815078234e-07,
      "logps/chosen": -283.63848876953125,
      "logps/rejected": -453.16619873046875,
      "loss": 0.1636,
      "rewards/chosen": 1.5939074754714966,
      "rewards/margins": 10.390283584594727,
      "rewards/rejected": -8.79637622833252,
      "step": 695
    },
    {
      "epoch": 0.45,
      "grad_norm": 19.494522094726562,
      "kl": 0.0,
      "learning_rate": 3.068990042674253e-07,
      "logps/chosen": -273.1427917480469,
      "logps/rejected": -421.3829650878906,
      "loss": 0.1513,
      "rewards/chosen": 1.827331304550171,
      "rewards/margins": 10.396958351135254,
      "rewards/rejected": -8.569626808166504,
      "step": 700
    },
    {
      "epoch": 0.45,
      "grad_norm": 23.5523624420166,
      "kl": 0.0,
      "learning_rate": 3.051209103840683e-07,
      "logps/chosen": -246.75692749023438,
      "logps/rejected": -434.19219970703125,
      "loss": 0.1616,
      "rewards/chosen": 1.8926893472671509,
      "rewards/margins": 10.59654712677002,
      "rewards/rejected": -8.703857421875,
      "step": 705
    },
    {
      "epoch": 0.45,
      "grad_norm": 25.01667022705078,
      "kl": 0.0,
      "learning_rate": 3.033428165007112e-07,
      "logps/chosen": -266.95745849609375,
      "logps/rejected": -419.81640625,
      "loss": 0.1713,
      "rewards/chosen": 1.3775568008422852,
      "rewards/margins": 9.493722915649414,
      "rewards/rejected": -8.116167068481445,
      "step": 710
    },
    {
      "epoch": 0.46,
      "grad_norm": 23.61768341064453,
      "kl": 0.0,
      "learning_rate": 3.015647226173542e-07,
      "logps/chosen": -261.8187561035156,
      "logps/rejected": -449.9925842285156,
      "loss": 0.15,
      "rewards/chosen": 1.6472257375717163,
      "rewards/margins": 10.508228302001953,
      "rewards/rejected": -8.861001968383789,
      "step": 715
    },
    {
      "epoch": 0.46,
      "grad_norm": 22.56501007080078,
      "kl": 0.0,
      "learning_rate": 2.9978662873399715e-07,
      "logps/chosen": -332.85504150390625,
      "logps/rejected": -378.87347412109375,
      "loss": 0.1669,
      "rewards/chosen": 1.8122440576553345,
      "rewards/margins": 10.017203330993652,
      "rewards/rejected": -8.20495891571045,
      "step": 720
    },
    {
      "epoch": 0.46,
      "grad_norm": 20.667505264282227,
      "kl": 0.0,
      "learning_rate": 2.9800853485064007e-07,
      "logps/chosen": -276.8742980957031,
      "logps/rejected": -414.7447204589844,
      "loss": 0.1585,
      "rewards/chosen": 1.9667861461639404,
      "rewards/margins": 10.267745971679688,
      "rewards/rejected": -8.300959587097168,
      "step": 725
    },
    {
      "epoch": 0.47,
      "grad_norm": 24.693187713623047,
      "kl": 0.0,
      "learning_rate": 2.9623044096728305e-07,
      "logps/chosen": -274.4808044433594,
      "logps/rejected": -429.78558349609375,
      "loss": 0.1546,
      "rewards/chosen": 1.594257116317749,
      "rewards/margins": 9.889884948730469,
      "rewards/rejected": -8.29562759399414,
      "step": 730
    },
    {
      "epoch": 0.47,
      "grad_norm": 30.14167022705078,
      "kl": 0.0,
      "learning_rate": 2.94452347083926e-07,
      "logps/chosen": -272.5168762207031,
      "logps/rejected": -415.374755859375,
      "loss": 0.1694,
      "rewards/chosen": 1.5294649600982666,
      "rewards/margins": 10.039915084838867,
      "rewards/rejected": -8.51045036315918,
      "step": 735
    },
    {
      "epoch": 0.47,
      "grad_norm": 22.99307632446289,
      "kl": 0.0,
      "learning_rate": 2.92674253200569e-07,
      "logps/chosen": -246.6560821533203,
      "logps/rejected": -418.26104736328125,
      "loss": 0.1584,
      "rewards/chosen": 1.566481113433838,
      "rewards/margins": 9.754480361938477,
      "rewards/rejected": -8.18799877166748,
      "step": 740
    },
    {
      "epoch": 0.48,
      "grad_norm": 19.63002586364746,
      "kl": 0.0,
      "learning_rate": 2.9089615931721197e-07,
      "logps/chosen": -289.92266845703125,
      "logps/rejected": -431.01336669921875,
      "loss": 0.152,
      "rewards/chosen": 1.7927067279815674,
      "rewards/margins": 9.82994270324707,
      "rewards/rejected": -8.037236213684082,
      "step": 745
    },
    {
      "epoch": 0.48,
      "grad_norm": 21.843643188476562,
      "kl": 0.0,
      "learning_rate": 2.8911806543385494e-07,
      "logps/chosen": -272.4095764160156,
      "logps/rejected": -384.18939208984375,
      "loss": 0.1658,
      "rewards/chosen": 1.6324083805084229,
      "rewards/margins": 9.954187393188477,
      "rewards/rejected": -8.321779251098633,
      "step": 750
    },
    {
      "epoch": 0.48,
      "grad_norm": 27.172988891601562,
      "kl": 0.0,
      "learning_rate": 2.873399715504978e-07,
      "logps/chosen": -260.12127685546875,
      "logps/rejected": -388.8544616699219,
      "loss": 0.163,
      "rewards/chosen": 1.7627140283584595,
      "rewards/margins": 10.121767044067383,
      "rewards/rejected": -8.359052658081055,
      "step": 755
    },
    {
      "epoch": 0.49,
      "grad_norm": 23.648014068603516,
      "kl": 0.0,
      "learning_rate": 2.855618776671408e-07,
      "logps/chosen": -262.92486572265625,
      "logps/rejected": -410.4188537597656,
      "loss": 0.1686,
      "rewards/chosen": 1.493422508239746,
      "rewards/margins": 9.909089088439941,
      "rewards/rejected": -8.415667533874512,
      "step": 760
    },
    {
      "epoch": 0.49,
      "grad_norm": 23.27745819091797,
      "kl": 0.0,
      "learning_rate": 2.8378378378378376e-07,
      "logps/chosen": -288.63421630859375,
      "logps/rejected": -406.29974365234375,
      "loss": 0.1504,
      "rewards/chosen": 1.8163105249404907,
      "rewards/margins": 10.033960342407227,
      "rewards/rejected": -8.217649459838867,
      "step": 765
    },
    {
      "epoch": 0.49,
      "grad_norm": 22.61689567565918,
      "kl": 0.0,
      "learning_rate": 2.8200568990042673e-07,
      "logps/chosen": -246.67105102539062,
      "logps/rejected": -425.2874450683594,
      "loss": 0.1575,
      "rewards/chosen": 1.4416098594665527,
      "rewards/margins": 10.138015747070312,
      "rewards/rejected": -8.696405410766602,
      "step": 770
    },
    {
      "epoch": 0.5,
      "grad_norm": 20.569795608520508,
      "kl": 0.0,
      "learning_rate": 2.802275960170697e-07,
      "logps/chosen": -291.814208984375,
      "logps/rejected": -399.8885192871094,
      "loss": 0.1491,
      "rewards/chosen": 1.771124243736267,
      "rewards/margins": 9.939413070678711,
      "rewards/rejected": -8.168289184570312,
      "step": 775
    },
    {
      "epoch": 0.5,
      "grad_norm": 27.513673782348633,
      "kl": 0.0,
      "learning_rate": 2.784495021337127e-07,
      "logps/chosen": -264.59918212890625,
      "logps/rejected": -426.15185546875,
      "loss": 0.1835,
      "rewards/chosen": 1.4606132507324219,
      "rewards/margins": 9.935892105102539,
      "rewards/rejected": -8.475278854370117,
      "step": 780
    },
    {
      "epoch": 0.5,
      "grad_norm": 25.520795822143555,
      "kl": 0.0,
      "learning_rate": 2.766714082503556e-07,
      "logps/chosen": -298.03466796875,
      "logps/rejected": -420.57635498046875,
      "loss": 0.1614,
      "rewards/chosen": 1.6418640613555908,
      "rewards/margins": 10.411275863647461,
      "rewards/rejected": -8.769411087036133,
      "step": 785
    },
    {
      "epoch": 0.51,
      "grad_norm": 25.321683883666992,
      "kl": 0.0,
      "learning_rate": 2.7489331436699857e-07,
      "logps/chosen": -278.0859069824219,
      "logps/rejected": -383.1441650390625,
      "loss": 0.1725,
      "rewards/chosen": 1.6416689157485962,
      "rewards/margins": 9.834056854248047,
      "rewards/rejected": -8.192388534545898,
      "step": 790
    },
    {
      "epoch": 0.51,
      "grad_norm": 18.801525115966797,
      "kl": 0.0,
      "learning_rate": 2.7311522048364154e-07,
      "logps/chosen": -293.8526306152344,
      "logps/rejected": -464.01580810546875,
      "loss": 0.1438,
      "rewards/chosen": 1.7998111248016357,
      "rewards/margins": 11.001166343688965,
      "rewards/rejected": -9.20135498046875,
      "step": 795
    },
    {
      "epoch": 0.51,
      "grad_norm": 31.995546340942383,
      "kl": 0.0,
      "learning_rate": 2.7133712660028446e-07,
      "logps/chosen": -266.13580322265625,
      "logps/rejected": -452.4052734375,
      "loss": 0.1547,
      "rewards/chosen": 1.85707688331604,
      "rewards/margins": 10.742141723632812,
      "rewards/rejected": -8.885065078735352,
      "step": 800
    },
    {
      "epoch": 0.52,
      "grad_norm": 28.300655364990234,
      "kl": 0.0,
      "learning_rate": 2.6955903271692744e-07,
      "logps/chosen": -276.36151123046875,
      "logps/rejected": -424.6527404785156,
      "loss": 0.1654,
      "rewards/chosen": 1.4731123447418213,
      "rewards/margins": 10.520303726196289,
      "rewards/rejected": -9.04719066619873,
      "step": 805
    },
    {
      "epoch": 0.52,
      "grad_norm": 21.696815490722656,
      "kl": 0.0,
      "learning_rate": 2.677809388335704e-07,
      "logps/chosen": -248.7739715576172,
      "logps/rejected": -468.64617919921875,
      "loss": 0.14,
      "rewards/chosen": 1.3544524908065796,
      "rewards/margins": 10.655045509338379,
      "rewards/rejected": -9.300593376159668,
      "step": 810
    },
    {
      "epoch": 0.52,
      "grad_norm": 26.037267684936523,
      "kl": 0.0,
      "learning_rate": 2.6600284495021333e-07,
      "logps/chosen": -278.1371765136719,
      "logps/rejected": -395.54315185546875,
      "loss": 0.1708,
      "rewards/chosen": 1.6430387496948242,
      "rewards/margins": 9.990675926208496,
      "rewards/rejected": -8.347638130187988,
      "step": 815
    },
    {
      "epoch": 0.52,
      "grad_norm": 27.351716995239258,
      "kl": 0.0,
      "learning_rate": 2.642247510668563e-07,
      "logps/chosen": -296.94207763671875,
      "logps/rejected": -418.74951171875,
      "loss": 0.159,
      "rewards/chosen": 1.9864873886108398,
      "rewards/margins": 11.01016902923584,
      "rewards/rejected": -9.023681640625,
      "step": 820
    },
    {
      "epoch": 0.53,
      "grad_norm": 27.478801727294922,
      "kl": 0.0,
      "learning_rate": 2.624466571834993e-07,
      "logps/chosen": -288.36370849609375,
      "logps/rejected": -431.9353942871094,
      "loss": 0.1483,
      "rewards/chosen": 2.1340115070343018,
      "rewards/margins": 11.531509399414062,
      "rewards/rejected": -9.397497177124023,
      "step": 825
    },
    {
      "epoch": 0.53,
      "grad_norm": 30.95049285888672,
      "kl": 0.0,
      "learning_rate": 2.6066856330014225e-07,
      "logps/chosen": -252.1579132080078,
      "logps/rejected": -424.7654724121094,
      "loss": 0.1578,
      "rewards/chosen": 1.2540996074676514,
      "rewards/margins": 10.03384780883789,
      "rewards/rejected": -8.77974796295166,
      "step": 830
    },
    {
      "epoch": 0.53,
      "grad_norm": 25.74098777770996,
      "kl": 0.0,
      "learning_rate": 2.5889046941678523e-07,
      "logps/chosen": -260.67315673828125,
      "logps/rejected": -398.57952880859375,
      "loss": 0.1554,
      "rewards/chosen": 1.9767462015151978,
      "rewards/margins": 10.994711875915527,
      "rewards/rejected": -9.017965316772461,
      "step": 835
    },
    {
      "epoch": 0.54,
      "grad_norm": 24.75625991821289,
      "kl": 0.0,
      "learning_rate": 2.5711237553342815e-07,
      "logps/chosen": -297.8096923828125,
      "logps/rejected": -454.16265869140625,
      "loss": 0.1601,
      "rewards/chosen": 1.8933601379394531,
      "rewards/margins": 10.956032752990723,
      "rewards/rejected": -9.062673568725586,
      "step": 840
    },
    {
      "epoch": 0.54,
      "grad_norm": 19.37633514404297,
      "kl": 0.0,
      "learning_rate": 2.5533428165007107e-07,
      "logps/chosen": -246.20755004882812,
      "logps/rejected": -417.218994140625,
      "loss": 0.1625,
      "rewards/chosen": 1.7149181365966797,
      "rewards/margins": 10.08183479309082,
      "rewards/rejected": -8.366915702819824,
      "step": 845
    },
    {
      "epoch": 0.54,
      "grad_norm": 22.729887008666992,
      "kl": 0.0,
      "learning_rate": 2.5355618776671404e-07,
      "logps/chosen": -295.47833251953125,
      "logps/rejected": -383.05230712890625,
      "loss": 0.1556,
      "rewards/chosen": 1.9083483219146729,
      "rewards/margins": 10.142326354980469,
      "rewards/rejected": -8.233978271484375,
      "step": 850
    },
    {
      "epoch": 0.55,
      "grad_norm": 20.790363311767578,
      "kl": 0.0,
      "learning_rate": 2.51778093883357e-07,
      "logps/chosen": -252.0298309326172,
      "logps/rejected": -439.210205078125,
      "loss": 0.1419,
      "rewards/chosen": 1.6935409307479858,
      "rewards/margins": 10.923540115356445,
      "rewards/rejected": -9.229999542236328,
      "step": 855
    },
    {
      "epoch": 0.55,
      "grad_norm": 20.501516342163086,
      "kl": 0.0,
      "learning_rate": 2.5e-07,
      "logps/chosen": -305.12030029296875,
      "logps/rejected": -444.84796142578125,
      "loss": 0.1369,
      "rewards/chosen": 1.9432222843170166,
      "rewards/margins": 11.106147766113281,
      "rewards/rejected": -9.162925720214844,
      "step": 860
    },
    {
      "epoch": 0.55,
      "grad_norm": 18.870508193969727,
      "kl": 0.0,
      "learning_rate": 2.4822190611664296e-07,
      "logps/chosen": -276.5811462402344,
      "logps/rejected": -418.9603576660156,
      "loss": 0.155,
      "rewards/chosen": 1.7197465896606445,
      "rewards/margins": 10.493929862976074,
      "rewards/rejected": -8.774185180664062,
      "step": 865
    },
    {
      "epoch": 0.56,
      "grad_norm": 29.126863479614258,
      "kl": 0.0,
      "learning_rate": 2.4644381223328594e-07,
      "logps/chosen": -265.5671691894531,
      "logps/rejected": -456.1929626464844,
      "loss": 0.1591,
      "rewards/chosen": 1.5650779008865356,
      "rewards/margins": 10.798846244812012,
      "rewards/rejected": -9.233770370483398,
      "step": 870
    },
    {
      "epoch": 0.56,
      "grad_norm": 24.848697662353516,
      "kl": 0.0,
      "learning_rate": 2.4466571834992886e-07,
      "logps/chosen": -254.2496337890625,
      "logps/rejected": -432.71038818359375,
      "loss": 0.1563,
      "rewards/chosen": 1.713888168334961,
      "rewards/margins": 10.464970588684082,
      "rewards/rejected": -8.751081466674805,
      "step": 875
    },
    {
      "epoch": 0.56,
      "grad_norm": 25.702816009521484,
      "kl": 0.0,
      "learning_rate": 2.4288762446657183e-07,
      "logps/chosen": -275.37030029296875,
      "logps/rejected": -427.5972595214844,
      "loss": 0.1629,
      "rewards/chosen": 1.6576436758041382,
      "rewards/margins": 10.351664543151855,
      "rewards/rejected": -8.69402027130127,
      "step": 880
    },
    {
      "epoch": 0.57,
      "grad_norm": 22.683273315429688,
      "kl": 0.0,
      "learning_rate": 2.411095305832148e-07,
      "logps/chosen": -282.93426513671875,
      "logps/rejected": -441.66064453125,
      "loss": 0.1249,
      "rewards/chosen": 1.9189395904541016,
      "rewards/margins": 11.167387008666992,
      "rewards/rejected": -9.248448371887207,
      "step": 885
    },
    {
      "epoch": 0.57,
      "grad_norm": 24.9903564453125,
      "kl": 0.0,
      "learning_rate": 2.393314366998578e-07,
      "logps/chosen": -294.500732421875,
      "logps/rejected": -424.95391845703125,
      "loss": 0.1396,
      "rewards/chosen": 1.8850313425064087,
      "rewards/margins": 10.923759460449219,
      "rewards/rejected": -9.038727760314941,
      "step": 890
    },
    {
      "epoch": 0.57,
      "grad_norm": 22.959312438964844,
      "kl": 0.0,
      "learning_rate": 2.375533428165007e-07,
      "logps/chosen": -276.75775146484375,
      "logps/rejected": -445.98052978515625,
      "loss": 0.1612,
      "rewards/chosen": 1.7431560754776,
      "rewards/margins": 10.924168586730957,
      "rewards/rejected": -9.181012153625488,
      "step": 895
    },
    {
      "epoch": 0.58,
      "grad_norm": 21.280902862548828,
      "kl": 0.0,
      "learning_rate": 2.3577524893314365e-07,
      "logps/chosen": -266.2899475097656,
      "logps/rejected": -419.53594970703125,
      "loss": 0.1366,
      "rewards/chosen": 2.116002321243286,
      "rewards/margins": 10.943870544433594,
      "rewards/rejected": -8.82786750793457,
      "step": 900
    },
    {
      "epoch": 0.58,
      "grad_norm": 23.134008407592773,
      "kl": 0.0,
      "learning_rate": 2.3399715504978662e-07,
      "logps/chosen": -295.0397033691406,
      "logps/rejected": -433.302978515625,
      "loss": 0.1485,
      "rewards/chosen": 1.6045787334442139,
      "rewards/margins": 10.714460372924805,
      "rewards/rejected": -9.109882354736328,
      "step": 905
    },
    {
      "epoch": 0.58,
      "grad_norm": 21.066099166870117,
      "kl": 0.0,
      "learning_rate": 2.322190611664296e-07,
      "logps/chosen": -242.4779815673828,
      "logps/rejected": -413.7901306152344,
      "loss": 0.1463,
      "rewards/chosen": 1.6984901428222656,
      "rewards/margins": 10.702180862426758,
      "rewards/rejected": -9.003690719604492,
      "step": 910
    },
    {
      "epoch": 0.59,
      "grad_norm": 23.61625099182129,
      "kl": 0.0,
      "learning_rate": 2.304409672830725e-07,
      "logps/chosen": -270.14599609375,
      "logps/rejected": -437.8699645996094,
      "loss": 0.1589,
      "rewards/chosen": 1.7722479104995728,
      "rewards/margins": 10.022378921508789,
      "rewards/rejected": -8.250131607055664,
      "step": 915
    },
    {
      "epoch": 0.59,
      "grad_norm": 22.70294952392578,
      "kl": 0.0,
      "learning_rate": 2.2866287339971549e-07,
      "logps/chosen": -285.99383544921875,
      "logps/rejected": -425.30511474609375,
      "loss": 0.1263,
      "rewards/chosen": 2.163151502609253,
      "rewards/margins": 10.922938346862793,
      "rewards/rejected": -8.759786605834961,
      "step": 920
    },
    {
      "epoch": 0.59,
      "grad_norm": 20.896127700805664,
      "kl": 0.0,
      "learning_rate": 2.2688477951635846e-07,
      "logps/chosen": -257.12646484375,
      "logps/rejected": -474.15789794921875,
      "loss": 0.1402,
      "rewards/chosen": 1.7868337631225586,
      "rewards/margins": 10.79575252532959,
      "rewards/rejected": -9.008918762207031,
      "step": 925
    },
    {
      "epoch": 0.6,
      "grad_norm": 20.20517921447754,
      "kl": 0.0,
      "learning_rate": 2.251066856330014e-07,
      "logps/chosen": -281.36016845703125,
      "logps/rejected": -409.21807861328125,
      "loss": 0.1405,
      "rewards/chosen": 1.9572327136993408,
      "rewards/margins": 10.755359649658203,
      "rewards/rejected": -8.798127174377441,
      "step": 930
    },
    {
      "epoch": 0.6,
      "grad_norm": 24.792766571044922,
      "kl": 0.0,
      "learning_rate": 2.2332859174964438e-07,
      "logps/chosen": -274.1449890136719,
      "logps/rejected": -440.4224548339844,
      "loss": 0.1612,
      "rewards/chosen": 1.4629716873168945,
      "rewards/margins": 10.279823303222656,
      "rewards/rejected": -8.816852569580078,
      "step": 935
    },
    {
      "epoch": 0.6,
      "grad_norm": 25.996437072753906,
      "kl": 0.0,
      "learning_rate": 2.2155049786628733e-07,
      "logps/chosen": -258.3365783691406,
      "logps/rejected": -455.66571044921875,
      "loss": 0.1593,
      "rewards/chosen": 1.8357813358306885,
      "rewards/margins": 11.063421249389648,
      "rewards/rejected": -9.227640151977539,
      "step": 940
    },
    {
      "epoch": 0.6,
      "grad_norm": 21.876049041748047,
      "kl": 0.0,
      "learning_rate": 2.1977240398293027e-07,
      "logps/chosen": -254.3147430419922,
      "logps/rejected": -438.74005126953125,
      "loss": 0.1293,
      "rewards/chosen": 1.6060841083526611,
      "rewards/margins": 10.991594314575195,
      "rewards/rejected": -9.38550853729248,
      "step": 945
    },
    {
      "epoch": 0.61,
      "grad_norm": 27.55910301208496,
      "kl": 0.0,
      "learning_rate": 2.1799431009957325e-07,
      "logps/chosen": -275.89837646484375,
      "logps/rejected": -442.4541931152344,
      "loss": 0.1545,
      "rewards/chosen": 1.7877737283706665,
      "rewards/margins": 11.597868919372559,
      "rewards/rejected": -9.81009578704834,
      "step": 950
    },
    {
      "epoch": 0.61,
      "grad_norm": 19.6149959564209,
      "kl": 0.0,
      "learning_rate": 2.1621621621621622e-07,
      "logps/chosen": -327.98602294921875,
      "logps/rejected": -448.33856201171875,
      "loss": 0.0995,
      "rewards/chosen": 2.8413889408111572,
      "rewards/margins": 11.999773025512695,
      "rewards/rejected": -9.158384323120117,
      "step": 955
    },
    {
      "epoch": 0.61,
      "grad_norm": 20.440847396850586,
      "kl": 0.0,
      "learning_rate": 2.1443812233285914e-07,
      "logps/chosen": -273.84930419921875,
      "logps/rejected": -407.75018310546875,
      "loss": 0.1382,
      "rewards/chosen": 2.0720973014831543,
      "rewards/margins": 10.910283088684082,
      "rewards/rejected": -8.83818531036377,
      "step": 960
    },
    {
      "epoch": 0.62,
      "grad_norm": 24.151046752929688,
      "kl": 0.0,
      "learning_rate": 2.1266002844950212e-07,
      "logps/chosen": -284.7245788574219,
      "logps/rejected": -433.08197021484375,
      "loss": 0.1285,
      "rewards/chosen": 2.2533042430877686,
      "rewards/margins": 11.679128646850586,
      "rewards/rejected": -9.425825119018555,
      "step": 965
    },
    {
      "epoch": 0.62,
      "grad_norm": 24.470195770263672,
      "kl": 0.0,
      "learning_rate": 2.108819345661451e-07,
      "logps/chosen": -320.61468505859375,
      "logps/rejected": -442.22869873046875,
      "loss": 0.1429,
      "rewards/chosen": 1.9059406518936157,
      "rewards/margins": 11.600639343261719,
      "rewards/rejected": -9.69469928741455,
      "step": 970
    },
    {
      "epoch": 0.62,
      "grad_norm": 26.81120491027832,
      "kl": 0.0,
      "learning_rate": 2.0910384068278806e-07,
      "logps/chosen": -257.6952209472656,
      "logps/rejected": -392.0545959472656,
      "loss": 0.1483,
      "rewards/chosen": 1.6184053421020508,
      "rewards/margins": 10.500303268432617,
      "rewards/rejected": -8.881898880004883,
      "step": 975
    },
    {
      "epoch": 0.63,
      "grad_norm": 19.685026168823242,
      "kl": 0.0,
      "learning_rate": 2.0732574679943098e-07,
      "logps/chosen": -305.773681640625,
      "logps/rejected": -393.2521057128906,
      "loss": 0.1456,
      "rewards/chosen": 2.0023820400238037,
      "rewards/margins": 11.047486305236816,
      "rewards/rejected": -9.045104026794434,
      "step": 980
    },
    {
      "epoch": 0.63,
      "grad_norm": 24.304468154907227,
      "kl": 0.0,
      "learning_rate": 2.0554765291607396e-07,
      "logps/chosen": -266.98773193359375,
      "logps/rejected": -409.9839782714844,
      "loss": 0.152,
      "rewards/chosen": 1.678897500038147,
      "rewards/margins": 10.779475212097168,
      "rewards/rejected": -9.100576400756836,
      "step": 985
    },
    {
      "epoch": 0.63,
      "grad_norm": 23.530902862548828,
      "kl": 0.0,
      "learning_rate": 2.0376955903271693e-07,
      "logps/chosen": -268.0154724121094,
      "logps/rejected": -446.49041748046875,
      "loss": 0.1365,
      "rewards/chosen": 1.426351547241211,
      "rewards/margins": 10.987870216369629,
      "rewards/rejected": -9.561517715454102,
      "step": 990
    },
    {
      "epoch": 0.64,
      "grad_norm": 25.060619354248047,
      "kl": 0.0,
      "learning_rate": 2.0199146514935988e-07,
      "logps/chosen": -304.2224426269531,
      "logps/rejected": -436.77734375,
      "loss": 0.1559,
      "rewards/chosen": 1.8927513360977173,
      "rewards/margins": 10.877464294433594,
      "rewards/rejected": -8.984713554382324,
      "step": 995
    },
    {
      "epoch": 0.64,
      "grad_norm": 26.11311149597168,
      "kl": 0.0,
      "learning_rate": 2.0021337126600283e-07,
      "logps/chosen": -263.1839599609375,
      "logps/rejected": -427.57110595703125,
      "loss": 0.1546,
      "rewards/chosen": 1.825537919998169,
      "rewards/margins": 10.585159301757812,
      "rewards/rejected": -8.759620666503906,
      "step": 1000
    },
    {
      "epoch": 0.64,
      "grad_norm": 21.942873001098633,
      "kl": 0.0,
      "learning_rate": 1.984352773826458e-07,
      "logps/chosen": -278.3871765136719,
      "logps/rejected": -397.9619140625,
      "loss": 0.1374,
      "rewards/chosen": 1.7600288391113281,
      "rewards/margins": 10.372520446777344,
      "rewards/rejected": -8.612492561340332,
      "step": 1005
    },
    {
      "epoch": 0.65,
      "grad_norm": 21.323123931884766,
      "kl": 0.0,
      "learning_rate": 1.9665718349928875e-07,
      "logps/chosen": -238.81234741210938,
      "logps/rejected": -444.0513610839844,
      "loss": 0.1392,
      "rewards/chosen": 1.7811119556427002,
      "rewards/margins": 10.868303298950195,
      "rewards/rejected": -9.087190628051758,
      "step": 1010
    },
    {
      "epoch": 0.65,
      "grad_norm": 23.634305953979492,
      "kl": 0.0,
      "learning_rate": 1.9487908961593172e-07,
      "logps/chosen": -254.67422485351562,
      "logps/rejected": -439.06842041015625,
      "loss": 0.1281,
      "rewards/chosen": 1.955566167831421,
      "rewards/margins": 11.299899101257324,
      "rewards/rejected": -9.344332695007324,
      "step": 1015
    },
    {
      "epoch": 0.65,
      "grad_norm": 23.24001121520996,
      "kl": 0.0,
      "learning_rate": 1.931009957325747e-07,
      "logps/chosen": -261.807861328125,
      "logps/rejected": -439.89434814453125,
      "loss": 0.1331,
      "rewards/chosen": 1.958783745765686,
      "rewards/margins": 10.877193450927734,
      "rewards/rejected": -8.91840934753418,
      "step": 1020
    },
    {
      "epoch": 0.66,
      "grad_norm": 21.924964904785156,
      "kl": 0.0,
      "learning_rate": 1.9132290184921761e-07,
      "logps/chosen": -232.3147735595703,
      "logps/rejected": -428.94415283203125,
      "loss": 0.1509,
      "rewards/chosen": 1.5595568418502808,
      "rewards/margins": 10.47365951538086,
      "rewards/rejected": -8.914102554321289,
      "step": 1025
    },
    {
      "epoch": 0.66,
      "grad_norm": 22.37010383605957,
      "kl": 0.0,
      "learning_rate": 1.895448079658606e-07,
      "logps/chosen": -310.47308349609375,
      "logps/rejected": -423.1930236816406,
      "loss": 0.1239,
      "rewards/chosen": 2.3751728534698486,
      "rewards/margins": 11.746635437011719,
      "rewards/rejected": -9.371461868286133,
      "step": 1030
    },
    {
      "epoch": 0.66,
      "grad_norm": 18.277446746826172,
      "kl": 0.0,
      "learning_rate": 1.8776671408250356e-07,
      "logps/chosen": -238.3054656982422,
      "logps/rejected": -409.2549743652344,
      "loss": 0.1534,
      "rewards/chosen": 2.0489933490753174,
      "rewards/margins": 10.762951850891113,
      "rewards/rejected": -8.713958740234375,
      "step": 1035
    },
    {
      "epoch": 0.67,
      "grad_norm": 23.205121994018555,
      "kl": 0.0,
      "learning_rate": 1.859886201991465e-07,
      "logps/chosen": -312.9981994628906,
      "logps/rejected": -412.560791015625,
      "loss": 0.1551,
      "rewards/chosen": 1.8553714752197266,
      "rewards/margins": 10.774530410766602,
      "rewards/rejected": -8.919158935546875,
      "step": 1040
    },
    {
      "epoch": 0.67,
      "grad_norm": 22.926227569580078,
      "kl": 0.0,
      "learning_rate": 1.8421052631578946e-07,
      "logps/chosen": -326.86187744140625,
      "logps/rejected": -425.36798095703125,
      "loss": 0.1256,
      "rewards/chosen": 2.2971606254577637,
      "rewards/margins": 11.508152961730957,
      "rewards/rejected": -9.210992813110352,
      "step": 1045
    },
    {
      "epoch": 0.67,
      "grad_norm": 19.493038177490234,
      "kl": 0.0,
      "learning_rate": 1.8243243243243243e-07,
      "logps/chosen": -293.5108337402344,
      "logps/rejected": -433.5061950683594,
      "loss": 0.1287,
      "rewards/chosen": 1.9984190464019775,
      "rewards/margins": 11.403292655944824,
      "rewards/rejected": -9.40487289428711,
      "step": 1050
    },
    {
      "epoch": 0.67,
      "grad_norm": 20.409494400024414,
      "kl": 0.0,
      "learning_rate": 1.8065433854907538e-07,
      "logps/chosen": -254.408935546875,
      "logps/rejected": -463.6639099121094,
      "loss": 0.1377,
      "rewards/chosen": 2.059924602508545,
      "rewards/margins": 11.853521347045898,
      "rewards/rejected": -9.793596267700195,
      "step": 1055
    },
    {
      "epoch": 0.68,
      "grad_norm": 24.266460418701172,
      "kl": 0.0,
      "learning_rate": 1.7887624466571835e-07,
      "logps/chosen": -272.4609375,
      "logps/rejected": -444.1348571777344,
      "loss": 0.1345,
      "rewards/chosen": 1.9829723834991455,
      "rewards/margins": 11.607297897338867,
      "rewards/rejected": -9.6243257522583,
      "step": 1060
    },
    {
      "epoch": 0.68,
      "grad_norm": 22.047306060791016,
      "kl": 0.0,
      "learning_rate": 1.770981507823613e-07,
      "logps/chosen": -298.2177429199219,
      "logps/rejected": -434.88232421875,
      "loss": 0.1425,
      "rewards/chosen": 1.9892578125,
      "rewards/margins": 11.173238754272461,
      "rewards/rejected": -9.183981895446777,
      "step": 1065
    },
    {
      "epoch": 0.68,
      "grad_norm": 17.649913787841797,
      "kl": 0.0,
      "learning_rate": 1.7532005689900424e-07,
      "logps/chosen": -275.7359619140625,
      "logps/rejected": -409.9220275878906,
      "loss": 0.1268,
      "rewards/chosen": 1.9714463949203491,
      "rewards/margins": 10.847058296203613,
      "rewards/rejected": -8.875612258911133,
      "step": 1070
    },
    {
      "epoch": 0.69,
      "grad_norm": 21.967309951782227,
      "kl": 0.0,
      "learning_rate": 1.7354196301564722e-07,
      "logps/chosen": -281.526123046875,
      "logps/rejected": -468.4922790527344,
      "loss": 0.1484,
      "rewards/chosen": 1.7273063659667969,
      "rewards/margins": 11.125950813293457,
      "rewards/rejected": -9.39864444732666,
      "step": 1075
    },
    {
      "epoch": 0.69,
      "grad_norm": 24.07299041748047,
      "kl": 0.0,
      "learning_rate": 1.717638691322902e-07,
      "logps/chosen": -288.5100402832031,
      "logps/rejected": -438.12579345703125,
      "loss": 0.1292,
      "rewards/chosen": 2.2782421112060547,
      "rewards/margins": 11.389139175415039,
      "rewards/rejected": -9.110896110534668,
      "step": 1080
    },
    {
      "epoch": 0.69,
      "grad_norm": 24.698833465576172,
      "kl": 0.0,
      "learning_rate": 1.6998577524893314e-07,
      "logps/chosen": -262.21307373046875,
      "logps/rejected": -432.73095703125,
      "loss": 0.1417,
      "rewards/chosen": 1.5888358354568481,
      "rewards/margins": 11.009684562683105,
      "rewards/rejected": -9.42085075378418,
      "step": 1085
    },
    {
      "epoch": 0.7,
      "grad_norm": 19.981353759765625,
      "kl": 0.0,
      "learning_rate": 1.6820768136557609e-07,
      "logps/chosen": -248.55294799804688,
      "logps/rejected": -417.9847717285156,
      "loss": 0.142,
      "rewards/chosen": 2.168029308319092,
      "rewards/margins": 11.213571548461914,
      "rewards/rejected": -9.04554271697998,
      "step": 1090
    },
    {
      "epoch": 0.7,
      "grad_norm": 25.718555450439453,
      "kl": 0.0,
      "learning_rate": 1.6642958748221906e-07,
      "logps/chosen": -310.0647888183594,
      "logps/rejected": -412.52001953125,
      "loss": 0.1359,
      "rewards/chosen": 2.2913904190063477,
      "rewards/margins": 11.509147644042969,
      "rewards/rejected": -9.217756271362305,
      "step": 1095
    },
    {
      "epoch": 0.7,
      "grad_norm": 20.378263473510742,
      "kl": 0.0,
      "learning_rate": 1.64651493598862e-07,
      "logps/chosen": -271.46343994140625,
      "logps/rejected": -405.5977478027344,
      "loss": 0.1325,
      "rewards/chosen": 2.2153549194335938,
      "rewards/margins": 11.555242538452148,
      "rewards/rejected": -9.339888572692871,
      "step": 1100
    },
    {
      "epoch": 0.71,
      "grad_norm": 23.4803466796875,
      "kl": 0.0,
      "learning_rate": 1.6287339971550498e-07,
      "logps/chosen": -292.6392822265625,
      "logps/rejected": -450.56121826171875,
      "loss": 0.1272,
      "rewards/chosen": 1.7447988986968994,
      "rewards/margins": 11.052331924438477,
      "rewards/rejected": -9.307534217834473,
      "step": 1105
    },
    {
      "epoch": 0.71,
      "grad_norm": 21.62887954711914,
      "kl": 0.0,
      "learning_rate": 1.6109530583214793e-07,
      "logps/chosen": -255.4695281982422,
      "logps/rejected": -440.318115234375,
      "loss": 0.1474,
      "rewards/chosen": 2.0870609283447266,
      "rewards/margins": 11.327590942382812,
      "rewards/rejected": -9.240530014038086,
      "step": 1110
    },
    {
      "epoch": 0.71,
      "grad_norm": 20.0389461517334,
      "kl": 0.0,
      "learning_rate": 1.5931721194879087e-07,
      "logps/chosen": -288.50909423828125,
      "logps/rejected": -440.1390075683594,
      "loss": 0.1267,
      "rewards/chosen": 2.4988930225372314,
      "rewards/margins": 12.02427864074707,
      "rewards/rejected": -9.525384902954102,
      "step": 1115
    },
    {
      "epoch": 0.72,
      "grad_norm": 27.30327606201172,
      "kl": 0.0,
      "learning_rate": 1.5753911806543385e-07,
      "logps/chosen": -315.9127197265625,
      "logps/rejected": -439.2584533691406,
      "loss": 0.1366,
      "rewards/chosen": 2.387932300567627,
      "rewards/margins": 11.112966537475586,
      "rewards/rejected": -8.725034713745117,
      "step": 1120
    },
    {
      "epoch": 0.72,
      "grad_norm": 22.370424270629883,
      "kl": 0.0,
      "learning_rate": 1.5576102418207682e-07,
      "logps/chosen": -296.82879638671875,
      "logps/rejected": -427.3119201660156,
      "loss": 0.1312,
      "rewards/chosen": 2.379429817199707,
      "rewards/margins": 11.889947891235352,
      "rewards/rejected": -9.510516166687012,
      "step": 1125
    },
    {
      "epoch": 0.72,
      "grad_norm": 22.60016441345215,
      "kl": 0.0,
      "learning_rate": 1.5398293029871974e-07,
      "logps/chosen": -284.07672119140625,
      "logps/rejected": -433.29278564453125,
      "loss": 0.1373,
      "rewards/chosen": 1.8526710271835327,
      "rewards/margins": 10.757834434509277,
      "rewards/rejected": -8.905162811279297,
      "step": 1130
    },
    {
      "epoch": 0.73,
      "grad_norm": 21.129493713378906,
      "kl": 0.0,
      "learning_rate": 1.5220483641536272e-07,
      "logps/chosen": -293.85552978515625,
      "logps/rejected": -427.25091552734375,
      "loss": 0.1265,
      "rewards/chosen": 2.3632965087890625,
      "rewards/margins": 11.906625747680664,
      "rewards/rejected": -9.543329238891602,
      "step": 1135
    },
    {
      "epoch": 0.73,
      "grad_norm": 19.694997787475586,
      "kl": 0.0,
      "learning_rate": 1.504267425320057e-07,
      "logps/chosen": -303.8923645019531,
      "logps/rejected": -465.65948486328125,
      "loss": 0.1168,
      "rewards/chosen": 2.5519447326660156,
      "rewards/margins": 12.214879989624023,
      "rewards/rejected": -9.662935256958008,
      "step": 1140
    },
    {
      "epoch": 0.73,
      "grad_norm": 22.003957748413086,
      "kl": 0.0,
      "learning_rate": 1.4864864864864866e-07,
      "logps/chosen": -275.05450439453125,
      "logps/rejected": -431.39459228515625,
      "loss": 0.1346,
      "rewards/chosen": 1.9555597305297852,
      "rewards/margins": 11.127113342285156,
      "rewards/rejected": -9.171553611755371,
      "step": 1145
    },
    {
      "epoch": 0.74,
      "grad_norm": 22.918169021606445,
      "kl": 0.0,
      "learning_rate": 1.4687055476529158e-07,
      "logps/chosen": -302.3306579589844,
      "logps/rejected": -442.83282470703125,
      "loss": 0.1249,
      "rewards/chosen": 2.11725115776062,
      "rewards/margins": 10.936277389526367,
      "rewards/rejected": -8.819025039672852,
      "step": 1150
    },
    {
      "epoch": 0.74,
      "grad_norm": 21.72097396850586,
      "kl": 0.0,
      "learning_rate": 1.4509246088193456e-07,
      "logps/chosen": -287.4019470214844,
      "logps/rejected": -401.7447814941406,
      "loss": 0.1338,
      "rewards/chosen": 2.2235841751098633,
      "rewards/margins": 10.831747055053711,
      "rewards/rejected": -8.608162879943848,
      "step": 1155
    },
    {
      "epoch": 0.74,
      "grad_norm": 22.042200088500977,
      "kl": 0.0,
      "learning_rate": 1.4331436699857753e-07,
      "logps/chosen": -278.91546630859375,
      "logps/rejected": -414.25885009765625,
      "loss": 0.1356,
      "rewards/chosen": 2.200042724609375,
      "rewards/margins": 11.185142517089844,
      "rewards/rejected": -8.985098838806152,
      "step": 1160
    },
    {
      "epoch": 0.75,
      "grad_norm": 17.934968948364258,
      "kl": 0.0,
      "learning_rate": 1.4153627311522048e-07,
      "logps/chosen": -308.2718200683594,
      "logps/rejected": -443.8597717285156,
      "loss": 0.1129,
      "rewards/chosen": 2.3466713428497314,
      "rewards/margins": 11.657171249389648,
      "rewards/rejected": -9.31049919128418,
      "step": 1165
    },
    {
      "epoch": 0.75,
      "grad_norm": 24.983243942260742,
      "kl": 0.0,
      "learning_rate": 1.3975817923186345e-07,
      "logps/chosen": -260.9937438964844,
      "logps/rejected": -389.94403076171875,
      "loss": 0.1388,
      "rewards/chosen": 2.287144184112549,
      "rewards/margins": 10.213197708129883,
      "rewards/rejected": -7.926053524017334,
      "step": 1170
    },
    {
      "epoch": 0.75,
      "grad_norm": 23.24259376525879,
      "kl": 0.0,
      "learning_rate": 1.379800853485064e-07,
      "logps/chosen": -270.7221374511719,
      "logps/rejected": -430.4684143066406,
      "loss": 0.1291,
      "rewards/chosen": 2.354933977127075,
      "rewards/margins": 11.62955379486084,
      "rewards/rejected": -9.274620056152344,
      "step": 1175
    },
    {
      "epoch": 0.75,
      "grad_norm": 21.11787986755371,
      "kl": 0.0,
      "learning_rate": 1.3620199146514935e-07,
      "logps/chosen": -304.3623962402344,
      "logps/rejected": -417.33929443359375,
      "loss": 0.1336,
      "rewards/chosen": 2.0863852500915527,
      "rewards/margins": 11.270166397094727,
      "rewards/rejected": -9.183780670166016,
      "step": 1180
    },
    {
      "epoch": 0.76,
      "grad_norm": 23.130449295043945,
      "kl": 0.0,
      "learning_rate": 1.3442389758179232e-07,
      "logps/chosen": -289.29156494140625,
      "logps/rejected": -438.5927734375,
      "loss": 0.1334,
      "rewards/chosen": 1.9771530628204346,
      "rewards/margins": 11.332016944885254,
      "rewards/rejected": -9.354864120483398,
      "step": 1185
    },
    {
      "epoch": 0.76,
      "grad_norm": 21.25185775756836,
      "kl": 0.0,
      "learning_rate": 1.326458036984353e-07,
      "logps/chosen": -246.78213500976562,
      "logps/rejected": -435.413818359375,
      "loss": 0.1494,
      "rewards/chosen": 1.6873725652694702,
      "rewards/margins": 10.664718627929688,
      "rewards/rejected": -8.977346420288086,
      "step": 1190
    },
    {
      "epoch": 0.76,
      "grad_norm": 16.545639038085938,
      "kl": 0.0,
      "learning_rate": 1.3086770981507821e-07,
      "logps/chosen": -270.4247741699219,
      "logps/rejected": -447.81787109375,
      "loss": 0.1282,
      "rewards/chosen": 2.250525951385498,
      "rewards/margins": 11.644302368164062,
      "rewards/rejected": -9.393776893615723,
      "step": 1195
    },
    {
      "epoch": 0.77,
      "grad_norm": 18.8116512298584,
      "kl": 0.0,
      "learning_rate": 1.290896159317212e-07,
      "logps/chosen": -315.2632751464844,
      "logps/rejected": -431.31005859375,
      "loss": 0.1056,
      "rewards/chosen": 2.4115493297576904,
      "rewards/margins": 11.539453506469727,
      "rewards/rejected": -9.127903938293457,
      "step": 1200
    },
    {
      "epoch": 0.77,
      "grad_norm": 22.677959442138672,
      "kl": 0.0,
      "learning_rate": 1.2731152204836416e-07,
      "logps/chosen": -250.3134307861328,
      "logps/rejected": -441.03948974609375,
      "loss": 0.1227,
      "rewards/chosen": 2.1084024906158447,
      "rewards/margins": 11.651433944702148,
      "rewards/rejected": -9.543031692504883,
      "step": 1205
    },
    {
      "epoch": 0.77,
      "grad_norm": 21.911409378051758,
      "kl": 0.0,
      "learning_rate": 1.255334281650071e-07,
      "logps/chosen": -237.79061889648438,
      "logps/rejected": -430.192138671875,
      "loss": 0.1288,
      "rewards/chosen": 2.0766196250915527,
      "rewards/margins": 11.207183837890625,
      "rewards/rejected": -9.13056468963623,
      "step": 1210
    },
    {
      "epoch": 0.78,
      "grad_norm": 21.959409713745117,
      "kl": 0.0,
      "learning_rate": 1.2375533428165005e-07,
      "logps/chosen": -308.5229187011719,
      "logps/rejected": -417.3135681152344,
      "loss": 0.1202,
      "rewards/chosen": 2.373558759689331,
      "rewards/margins": 11.75732135772705,
      "rewards/rejected": -9.383761405944824,
      "step": 1215
    },
    {
      "epoch": 0.78,
      "grad_norm": 20.8580265045166,
      "kl": 0.0,
      "learning_rate": 1.2197724039829303e-07,
      "logps/chosen": -243.6355438232422,
      "logps/rejected": -430.52801513671875,
      "loss": 0.1265,
      "rewards/chosen": 2.0868420600891113,
      "rewards/margins": 11.455188751220703,
      "rewards/rejected": -9.36834716796875,
      "step": 1220
    },
    {
      "epoch": 0.78,
      "grad_norm": 17.877643585205078,
      "kl": 0.0,
      "learning_rate": 1.2019914651493598e-07,
      "logps/chosen": -261.396484375,
      "logps/rejected": -397.22882080078125,
      "loss": 0.1269,
      "rewards/chosen": 2.2846105098724365,
      "rewards/margins": 11.026288986206055,
      "rewards/rejected": -8.741678237915039,
      "step": 1225
    },
    {
      "epoch": 0.79,
      "grad_norm": 20.07047462463379,
      "kl": 0.0,
      "learning_rate": 1.1842105263157894e-07,
      "logps/chosen": -252.1815643310547,
      "logps/rejected": -434.57977294921875,
      "loss": 0.1224,
      "rewards/chosen": 2.228175640106201,
      "rewards/margins": 11.402864456176758,
      "rewards/rejected": -9.174688339233398,
      "step": 1230
    },
    {
      "epoch": 0.79,
      "grad_norm": 22.779888153076172,
      "kl": 0.0,
      "learning_rate": 1.166429587482219e-07,
      "logps/chosen": -310.24267578125,
      "logps/rejected": -429.3841857910156,
      "loss": 0.132,
      "rewards/chosen": 2.069182872772217,
      "rewards/margins": 11.373079299926758,
      "rewards/rejected": -9.3038969039917,
      "step": 1235
    },
    {
      "epoch": 0.79,
      "grad_norm": 20.758487701416016,
      "kl": 0.0,
      "learning_rate": 1.1486486486486487e-07,
      "logps/chosen": -254.3895263671875,
      "logps/rejected": -465.3462829589844,
      "loss": 0.1237,
      "rewards/chosen": 2.068655252456665,
      "rewards/margins": 11.601139068603516,
      "rewards/rejected": -9.532483100891113,
      "step": 1240
    },
    {
      "epoch": 0.8,
      "grad_norm": 19.828561782836914,
      "kl": 0.0,
      "learning_rate": 1.1308677098150782e-07,
      "logps/chosen": -282.51617431640625,
      "logps/rejected": -437.31103515625,
      "loss": 0.1296,
      "rewards/chosen": 2.1852059364318848,
      "rewards/margins": 11.828689575195312,
      "rewards/rejected": -9.64348316192627,
      "step": 1245
    },
    {
      "epoch": 0.8,
      "grad_norm": 19.10045623779297,
      "kl": 0.0,
      "learning_rate": 1.1130867709815078e-07,
      "logps/chosen": -290.7330017089844,
      "logps/rejected": -437.0845642089844,
      "loss": 0.1163,
      "rewards/chosen": 2.1793453693389893,
      "rewards/margins": 11.713521003723145,
      "rewards/rejected": -9.534174919128418,
      "step": 1250
    },
    {
      "epoch": 0.8,
      "grad_norm": 22.69818878173828,
      "kl": 0.0,
      "learning_rate": 1.0953058321479374e-07,
      "logps/chosen": -288.8890075683594,
      "logps/rejected": -411.38629150390625,
      "loss": 0.1308,
      "rewards/chosen": 2.0378472805023193,
      "rewards/margins": 10.46517276763916,
      "rewards/rejected": -8.427325248718262,
      "step": 1255
    },
    {
      "epoch": 0.81,
      "grad_norm": 21.247556686401367,
      "kl": 0.0,
      "learning_rate": 1.077524893314367e-07,
      "logps/chosen": -283.88751220703125,
      "logps/rejected": -429.2752990722656,
      "loss": 0.119,
      "rewards/chosen": 2.1524832248687744,
      "rewards/margins": 11.539883613586426,
      "rewards/rejected": -9.387399673461914,
      "step": 1260
    },
    {
      "epoch": 0.81,
      "grad_norm": 21.65992546081543,
      "kl": 0.0,
      "learning_rate": 1.0597439544807964e-07,
      "logps/chosen": -278.073486328125,
      "logps/rejected": -448.2752990722656,
      "loss": 0.1171,
      "rewards/chosen": 2.266263484954834,
      "rewards/margins": 12.238935470581055,
      "rewards/rejected": -9.972672462463379,
      "step": 1265
    },
    {
      "epoch": 0.81,
      "grad_norm": 19.32549285888672,
      "kl": 0.0,
      "learning_rate": 1.0419630156472262e-07,
      "logps/chosen": -303.88616943359375,
      "logps/rejected": -435.61810302734375,
      "loss": 0.1139,
      "rewards/chosen": 2.234591007232666,
      "rewards/margins": 11.52080249786377,
      "rewards/rejected": -9.286211013793945,
      "step": 1270
    },
    {
      "epoch": 0.82,
      "grad_norm": 20.137916564941406,
      "kl": 0.0,
      "learning_rate": 1.0241820768136557e-07,
      "logps/chosen": -281.22686767578125,
      "logps/rejected": -411.6228942871094,
      "loss": 0.1232,
      "rewards/chosen": 2.257664203643799,
      "rewards/margins": 11.2540864944458,
      "rewards/rejected": -8.996421813964844,
      "step": 1275
    },
    {
      "epoch": 0.82,
      "grad_norm": 15.928502082824707,
      "kl": 0.0,
      "learning_rate": 1.0064011379800854e-07,
      "logps/chosen": -316.901611328125,
      "logps/rejected": -428.92822265625,
      "loss": 0.1213,
      "rewards/chosen": 2.580500841140747,
      "rewards/margins": 11.729310989379883,
      "rewards/rejected": -9.148810386657715,
      "step": 1280
    },
    {
      "epoch": 0.82,
      "grad_norm": 24.323083877563477,
      "kl": 0.0,
      "learning_rate": 9.886201991465149e-08,
      "logps/chosen": -261.3116149902344,
      "logps/rejected": -456.38201904296875,
      "loss": 0.1232,
      "rewards/chosen": 1.9357669353485107,
      "rewards/margins": 11.504796981811523,
      "rewards/rejected": -9.569029808044434,
      "step": 1285
    },
    {
      "epoch": 0.83,
      "grad_norm": 19.80019760131836,
      "kl": 0.0,
      "learning_rate": 9.708392603129445e-08,
      "logps/chosen": -286.7679748535156,
      "logps/rejected": -409.57220458984375,
      "loss": 0.1177,
      "rewards/chosen": 1.9313856363296509,
      "rewards/margins": 11.41652774810791,
      "rewards/rejected": -9.48514175415039,
      "step": 1290
    },
    {
      "epoch": 0.83,
      "grad_norm": 15.446332931518555,
      "kl": 0.0,
      "learning_rate": 9.530583214793741e-08,
      "logps/chosen": -256.6762390136719,
      "logps/rejected": -445.1065368652344,
      "loss": 0.1037,
      "rewards/chosen": 2.22512149810791,
      "rewards/margins": 12.185127258300781,
      "rewards/rejected": -9.960004806518555,
      "step": 1295
    },
    {
      "epoch": 0.83,
      "grad_norm": 17.317325592041016,
      "kl": 0.0,
      "learning_rate": 9.352773826458037e-08,
      "logps/chosen": -248.8364715576172,
      "logps/rejected": -417.3561096191406,
      "loss": 0.1064,
      "rewards/chosen": 2.178281307220459,
      "rewards/margins": 12.139802932739258,
      "rewards/rejected": -9.96152114868164,
      "step": 1300
    },
    {
      "epoch": 0.83,
      "grad_norm": 16.94424057006836,
      "kl": 0.0,
      "learning_rate": 9.174964438122331e-08,
      "logps/chosen": -253.8694305419922,
      "logps/rejected": -388.809326171875,
      "loss": 0.1238,
      "rewards/chosen": 1.9573352336883545,
      "rewards/margins": 11.183786392211914,
      "rewards/rejected": -9.22645092010498,
      "step": 1305
    },
    {
      "epoch": 0.84,
      "grad_norm": 19.46892738342285,
      "kl": 0.0,
      "learning_rate": 8.997155049786629e-08,
      "logps/chosen": -299.6037902832031,
      "logps/rejected": -408.68701171875,
      "loss": 0.1313,
      "rewards/chosen": 2.082526683807373,
      "rewards/margins": 11.771112442016602,
      "rewards/rejected": -9.688586235046387,
      "step": 1310
    },
    {
      "epoch": 0.84,
      "grad_norm": 21.77025604248047,
      "kl": 0.0,
      "learning_rate": 8.819345661450925e-08,
      "logps/chosen": -276.5013122558594,
      "logps/rejected": -462.1937561035156,
      "loss": 0.1368,
      "rewards/chosen": 1.8290278911590576,
      "rewards/margins": 11.848515510559082,
      "rewards/rejected": -10.019487380981445,
      "step": 1315
    },
    {
      "epoch": 0.84,
      "grad_norm": 20.317564010620117,
      "kl": 0.0,
      "learning_rate": 8.64153627311522e-08,
      "logps/chosen": -275.123046875,
      "logps/rejected": -400.65899658203125,
      "loss": 0.1178,
      "rewards/chosen": 2.3277885913848877,
      "rewards/margins": 11.247326850891113,
      "rewards/rejected": -8.919538497924805,
      "step": 1320
    },
    {
      "epoch": 0.85,
      "grad_norm": 18.717248916625977,
      "kl": 0.0,
      "learning_rate": 8.463726884779517e-08,
      "logps/chosen": -261.8305358886719,
      "logps/rejected": -415.6966857910156,
      "loss": 0.1082,
      "rewards/chosen": 2.135937452316284,
      "rewards/margins": 11.28874683380127,
      "rewards/rejected": -9.152809143066406,
      "step": 1325
    },
    {
      "epoch": 0.85,
      "grad_norm": 21.309633255004883,
      "kl": 0.0,
      "learning_rate": 8.285917496443812e-08,
      "logps/chosen": -231.76626586914062,
      "logps/rejected": -448.5263671875,
      "loss": 0.1118,
      "rewards/chosen": 2.206472873687744,
      "rewards/margins": 12.001043319702148,
      "rewards/rejected": -9.794569969177246,
      "step": 1330
    },
    {
      "epoch": 0.85,
      "grad_norm": 19.966053009033203,
      "kl": 0.0,
      "learning_rate": 8.108108108108108e-08,
      "logps/chosen": -287.36712646484375,
      "logps/rejected": -427.26898193359375,
      "loss": 0.1209,
      "rewards/chosen": 2.056323528289795,
      "rewards/margins": 10.724042892456055,
      "rewards/rejected": -8.667718887329102,
      "step": 1335
    },
    {
      "epoch": 0.86,
      "grad_norm": 22.85637855529785,
      "kl": 0.0,
      "learning_rate": 7.930298719772404e-08,
      "logps/chosen": -253.1752471923828,
      "logps/rejected": -414.54827880859375,
      "loss": 0.1332,
      "rewards/chosen": 1.9090397357940674,
      "rewards/margins": 11.734771728515625,
      "rewards/rejected": -9.825732231140137,
      "step": 1340
    },
    {
      "epoch": 0.86,
      "grad_norm": 17.524154663085938,
      "kl": 0.0,
      "learning_rate": 7.7524893314367e-08,
      "logps/chosen": -271.58856201171875,
      "logps/rejected": -431.1787109375,
      "loss": 0.1257,
      "rewards/chosen": 2.0929927825927734,
      "rewards/margins": 11.595911026000977,
      "rewards/rejected": -9.502918243408203,
      "step": 1345
    },
    {
      "epoch": 0.86,
      "grad_norm": 20.388002395629883,
      "kl": 0.0,
      "learning_rate": 7.574679943100994e-08,
      "logps/chosen": -294.2145080566406,
      "logps/rejected": -427.0972595214844,
      "loss": 0.132,
      "rewards/chosen": 2.2338738441467285,
      "rewards/margins": 11.665974617004395,
      "rewards/rejected": -9.432100296020508,
      "step": 1350
    },
    {
      "epoch": 0.87,
      "grad_norm": 27.609825134277344,
      "kl": 0.0,
      "learning_rate": 7.396870554765292e-08,
      "logps/chosen": -277.4027404785156,
      "logps/rejected": -441.5484924316406,
      "loss": 0.1205,
      "rewards/chosen": 1.9443111419677734,
      "rewards/margins": 11.26426887512207,
      "rewards/rejected": -9.319957733154297,
      "step": 1355
    },
    {
      "epoch": 0.87,
      "grad_norm": 21.430208206176758,
      "kl": 0.0,
      "learning_rate": 7.219061166429587e-08,
      "logps/chosen": -232.32846069335938,
      "logps/rejected": -424.41668701171875,
      "loss": 0.143,
      "rewards/chosen": 2.0610389709472656,
      "rewards/margins": 10.865748405456543,
      "rewards/rejected": -8.804708480834961,
      "step": 1360
    },
    {
      "epoch": 0.87,
      "grad_norm": 17.94335174560547,
      "kl": 0.0,
      "learning_rate": 7.041251778093883e-08,
      "logps/chosen": -246.70504760742188,
      "logps/rejected": -452.76953125,
      "loss": 0.0997,
      "rewards/chosen": 2.304919481277466,
      "rewards/margins": 11.7770414352417,
      "rewards/rejected": -9.472122192382812,
      "step": 1365
    },
    {
      "epoch": 0.88,
      "grad_norm": 22.950284957885742,
      "kl": 0.0,
      "learning_rate": 6.863442389758179e-08,
      "logps/chosen": -267.05242919921875,
      "logps/rejected": -420.9137268066406,
      "loss": 0.1369,
      "rewards/chosen": 1.899043083190918,
      "rewards/margins": 11.120938301086426,
      "rewards/rejected": -9.221895217895508,
      "step": 1370
    },
    {
      "epoch": 0.88,
      "grad_norm": 18.250551223754883,
      "kl": 0.0,
      "learning_rate": 6.685633001422475e-08,
      "logps/chosen": -231.0555877685547,
      "logps/rejected": -449.4051818847656,
      "loss": 0.13,
      "rewards/chosen": 2.14901065826416,
      "rewards/margins": 11.469152450561523,
      "rewards/rejected": -9.320141792297363,
      "step": 1375
    },
    {
      "epoch": 0.88,
      "grad_norm": 20.363611221313477,
      "kl": 0.0,
      "learning_rate": 6.507823613086771e-08,
      "logps/chosen": -278.7294006347656,
      "logps/rejected": -418.37030029296875,
      "loss": 0.1272,
      "rewards/chosen": 2.073495626449585,
      "rewards/margins": 10.949335098266602,
      "rewards/rejected": -8.875838279724121,
      "step": 1380
    },
    {
      "epoch": 0.89,
      "grad_norm": 18.06746482849121,
      "kl": 0.0,
      "learning_rate": 6.330014224751067e-08,
      "logps/chosen": -283.6380920410156,
      "logps/rejected": -453.05987548828125,
      "loss": 0.1107,
      "rewards/chosen": 2.3998241424560547,
      "rewards/margins": 12.352110862731934,
      "rewards/rejected": -9.952287673950195,
      "step": 1385
    },
    {
      "epoch": 0.89,
      "grad_norm": 20.415699005126953,
      "kl": 0.0,
      "learning_rate": 6.152204836415363e-08,
      "logps/chosen": -305.8134765625,
      "logps/rejected": -397.2032775878906,
      "loss": 0.1191,
      "rewards/chosen": 2.6986749172210693,
      "rewards/margins": 11.584226608276367,
      "rewards/rejected": -8.885551452636719,
      "step": 1390
    },
    {
      "epoch": 0.89,
      "grad_norm": 20.92513084411621,
      "kl": 0.0,
      "learning_rate": 5.974395448079659e-08,
      "logps/chosen": -295.1770935058594,
      "logps/rejected": -471.5146484375,
      "loss": 0.1234,
      "rewards/chosen": 1.9177255630493164,
      "rewards/margins": 11.333423614501953,
      "rewards/rejected": -9.415698051452637,
      "step": 1395
    },
    {
      "epoch": 0.9,
      "grad_norm": 21.794998168945312,
      "kl": 0.0,
      "learning_rate": 5.796586059743954e-08,
      "logps/chosen": -271.8743591308594,
      "logps/rejected": -505.2965393066406,
      "loss": 0.1268,
      "rewards/chosen": 1.8860387802124023,
      "rewards/margins": 12.0774507522583,
      "rewards/rejected": -10.191411972045898,
      "step": 1400
    },
    {
      "epoch": 0.9,
      "grad_norm": 21.202049255371094,
      "kl": 0.0,
      "learning_rate": 5.61877667140825e-08,
      "logps/chosen": -277.30548095703125,
      "logps/rejected": -465.62744140625,
      "loss": 0.1104,
      "rewards/chosen": 2.4026882648468018,
      "rewards/margins": 12.06915283203125,
      "rewards/rejected": -9.666464805603027,
      "step": 1405
    },
    {
      "epoch": 0.9,
      "grad_norm": 23.27330780029297,
      "kl": 0.0,
      "learning_rate": 5.4409672830725456e-08,
      "logps/chosen": -276.54815673828125,
      "logps/rejected": -438.12005615234375,
      "loss": 0.1114,
      "rewards/chosen": 2.2622342109680176,
      "rewards/margins": 11.430865287780762,
      "rewards/rejected": -9.168631553649902,
      "step": 1410
    },
    {
      "epoch": 0.91,
      "grad_norm": 22.490863800048828,
      "kl": 0.0,
      "learning_rate": 5.2631578947368416e-08,
      "logps/chosen": -275.0463562011719,
      "logps/rejected": -461.47552490234375,
      "loss": 0.1348,
      "rewards/chosen": 2.0476670265197754,
      "rewards/margins": 12.236173629760742,
      "rewards/rejected": -10.188508033752441,
      "step": 1415
    },
    {
      "epoch": 0.91,
      "grad_norm": 22.146873474121094,
      "kl": 0.0,
      "learning_rate": 5.0853485064011376e-08,
      "logps/chosen": -285.8138122558594,
      "logps/rejected": -442.80169677734375,
      "loss": 0.114,
      "rewards/chosen": 2.3625457286834717,
      "rewards/margins": 11.950872421264648,
      "rewards/rejected": -9.588325500488281,
      "step": 1420
    },
    {
      "epoch": 0.91,
      "grad_norm": 18.684457778930664,
      "kl": 0.0,
      "learning_rate": 4.9075391180654337e-08,
      "logps/chosen": -281.9924011230469,
      "logps/rejected": -444.9959411621094,
      "loss": 0.1139,
      "rewards/chosen": 2.2367939949035645,
      "rewards/margins": 11.815010070800781,
      "rewards/rejected": -9.578217506408691,
      "step": 1425
    },
    {
      "epoch": 0.91,
      "grad_norm": 24.193016052246094,
      "kl": 0.0,
      "learning_rate": 4.72972972972973e-08,
      "logps/chosen": -319.1490783691406,
      "logps/rejected": -415.7984924316406,
      "loss": 0.1288,
      "rewards/chosen": 2.3501713275909424,
      "rewards/margins": 11.565084457397461,
      "rewards/rejected": -9.214912414550781,
      "step": 1430
    },
    {
      "epoch": 0.92,
      "grad_norm": 20.57202911376953,
      "kl": 0.0,
      "learning_rate": 4.551920341394026e-08,
      "logps/chosen": -267.69696044921875,
      "logps/rejected": -447.53302001953125,
      "loss": 0.139,
      "rewards/chosen": 2.0053253173828125,
      "rewards/margins": 11.46650218963623,
      "rewards/rejected": -9.461176872253418,
      "step": 1435
    },
    {
      "epoch": 0.92,
      "grad_norm": 20.329166412353516,
      "kl": 0.0,
      "learning_rate": 4.374110953058322e-08,
      "logps/chosen": -264.3152160644531,
      "logps/rejected": -474.037353515625,
      "loss": 0.117,
      "rewards/chosen": 2.193683624267578,
      "rewards/margins": 12.409022331237793,
      "rewards/rejected": -10.215338706970215,
      "step": 1440
    },
    {
      "epoch": 0.92,
      "grad_norm": 22.352367401123047,
      "kl": 0.0,
      "learning_rate": 4.196301564722617e-08,
      "logps/chosen": -286.8226623535156,
      "logps/rejected": -478.8553771972656,
      "loss": 0.1177,
      "rewards/chosen": 1.9940738677978516,
      "rewards/margins": 12.453526496887207,
      "rewards/rejected": -10.459451675415039,
      "step": 1445
    },
    {
      "epoch": 0.93,
      "grad_norm": 18.939308166503906,
      "kl": 0.0,
      "learning_rate": 4.018492176386913e-08,
      "logps/chosen": -276.21746826171875,
      "logps/rejected": -413.55389404296875,
      "loss": 0.1328,
      "rewards/chosen": 2.231210708618164,
      "rewards/margins": 11.543118476867676,
      "rewards/rejected": -9.311908721923828,
      "step": 1450
    },
    {
      "epoch": 0.93,
      "grad_norm": 18.116722106933594,
      "kl": 0.0,
      "learning_rate": 3.840682788051209e-08,
      "logps/chosen": -240.20028686523438,
      "logps/rejected": -416.8455505371094,
      "loss": 0.1142,
      "rewards/chosen": 2.345201253890991,
      "rewards/margins": 11.189725875854492,
      "rewards/rejected": -8.844523429870605,
      "step": 1455
    },
    {
      "epoch": 0.93,
      "grad_norm": 25.406269073486328,
      "kl": 0.0,
      "learning_rate": 3.6628733997155046e-08,
      "logps/chosen": -254.66464233398438,
      "logps/rejected": -418.0938415527344,
      "loss": 0.1186,
      "rewards/chosen": 2.0774874687194824,
      "rewards/margins": 11.201638221740723,
      "rewards/rejected": -9.124151229858398,
      "step": 1460
    },
    {
      "epoch": 0.94,
      "grad_norm": 18.636796951293945,
      "kl": 0.0,
      "learning_rate": 3.4850640113798006e-08,
      "logps/chosen": -275.15472412109375,
      "logps/rejected": -442.4652404785156,
      "loss": 0.1126,
      "rewards/chosen": 2.310152053833008,
      "rewards/margins": 12.194239616394043,
      "rewards/rejected": -9.884087562561035,
      "step": 1465
    },
    {
      "epoch": 0.94,
      "grad_norm": 27.727458953857422,
      "kl": 0.0,
      "learning_rate": 3.3072546230440967e-08,
      "logps/chosen": -282.36627197265625,
      "logps/rejected": -417.9161071777344,
      "loss": 0.1169,
      "rewards/chosen": 2.358586549758911,
      "rewards/margins": 11.822304725646973,
      "rewards/rejected": -9.463717460632324,
      "step": 1470
    },
    {
      "epoch": 0.94,
      "grad_norm": 18.799943923950195,
      "kl": 0.0,
      "learning_rate": 3.129445234708392e-08,
      "logps/chosen": -282.34100341796875,
      "logps/rejected": -400.6058044433594,
      "loss": 0.1164,
      "rewards/chosen": 2.1689610481262207,
      "rewards/margins": 11.757662773132324,
      "rewards/rejected": -9.588701248168945,
      "step": 1475
    },
    {
      "epoch": 0.95,
      "grad_norm": 18.94513702392578,
      "kl": 0.0,
      "learning_rate": 2.9516358463726884e-08,
      "logps/chosen": -265.2249755859375,
      "logps/rejected": -420.23870849609375,
      "loss": 0.1276,
      "rewards/chosen": 2.1199347972869873,
      "rewards/margins": 11.105647087097168,
      "rewards/rejected": -8.985712051391602,
      "step": 1480
    },
    {
      "epoch": 0.95,
      "grad_norm": 17.380094528198242,
      "kl": 0.0,
      "learning_rate": 2.7738264580369844e-08,
      "logps/chosen": -291.02392578125,
      "logps/rejected": -407.71038818359375,
      "loss": 0.1115,
      "rewards/chosen": 2.1979808807373047,
      "rewards/margins": 11.310641288757324,
      "rewards/rejected": -9.11266040802002,
      "step": 1485
    },
    {
      "epoch": 0.95,
      "grad_norm": 19.2188720703125,
      "kl": 0.0,
      "learning_rate": 2.59601706970128e-08,
      "logps/chosen": -280.51495361328125,
      "logps/rejected": -436.34893798828125,
      "loss": 0.1182,
      "rewards/chosen": 2.2310471534729004,
      "rewards/margins": 10.99431324005127,
      "rewards/rejected": -8.763265609741211,
      "step": 1490
    },
    {
      "epoch": 0.96,
      "grad_norm": 23.651437759399414,
      "kl": 0.0,
      "learning_rate": 2.418207681365576e-08,
      "logps/chosen": -260.20220947265625,
      "logps/rejected": -447.58489990234375,
      "loss": 0.1224,
      "rewards/chosen": 1.9462487697601318,
      "rewards/margins": 11.421347618103027,
      "rewards/rejected": -9.475099563598633,
      "step": 1495
    },
    {
      "epoch": 0.96,
      "grad_norm": 24.50865936279297,
      "kl": 0.0,
      "learning_rate": 2.240398293029872e-08,
      "logps/chosen": -273.6228942871094,
      "logps/rejected": -388.49114990234375,
      "loss": 0.1235,
      "rewards/chosen": 2.335383415222168,
      "rewards/margins": 10.98925495147705,
      "rewards/rejected": -8.653871536254883,
      "step": 1500
    },
    {
      "epoch": 0.96,
      "grad_norm": 17.999431610107422,
      "kl": 0.0,
      "learning_rate": 2.0625889046941676e-08,
      "logps/chosen": -287.52899169921875,
      "logps/rejected": -439.962646484375,
      "loss": 0.1142,
      "rewards/chosen": 2.2732601165771484,
      "rewards/margins": 11.966636657714844,
      "rewards/rejected": -9.693376541137695,
      "step": 1505
    },
    {
      "epoch": 0.97,
      "grad_norm": 22.014801025390625,
      "kl": 0.0,
      "learning_rate": 1.8847795163584636e-08,
      "logps/chosen": -259.125,
      "logps/rejected": -452.64752197265625,
      "loss": 0.1162,
      "rewards/chosen": 2.1289539337158203,
      "rewards/margins": 11.959836959838867,
      "rewards/rejected": -9.830883026123047,
      "step": 1510
    },
    {
      "epoch": 0.97,
      "grad_norm": 24.860143661499023,
      "kl": 0.0,
      "learning_rate": 1.7069701280227596e-08,
      "logps/chosen": -233.20724487304688,
      "logps/rejected": -443.26153564453125,
      "loss": 0.1203,
      "rewards/chosen": 1.8372972011566162,
      "rewards/margins": 11.322072982788086,
      "rewards/rejected": -9.484776496887207,
      "step": 1515
    },
    {
      "epoch": 0.97,
      "grad_norm": 16.33979034423828,
      "kl": 0.0,
      "learning_rate": 1.5291607396870554e-08,
      "logps/chosen": -242.3277587890625,
      "logps/rejected": -427.3155822753906,
      "loss": 0.1275,
      "rewards/chosen": 2.065825939178467,
      "rewards/margins": 11.441205978393555,
      "rewards/rejected": -9.375380516052246,
      "step": 1520
    },
    {
      "epoch": 0.98,
      "grad_norm": 22.828136444091797,
      "kl": 0.0,
      "learning_rate": 1.3513513513513514e-08,
      "logps/chosen": -296.04815673828125,
      "logps/rejected": -441.4925842285156,
      "loss": 0.1139,
      "rewards/chosen": 2.2007675170898438,
      "rewards/margins": 11.669252395629883,
      "rewards/rejected": -9.468484878540039,
      "step": 1525
    },
    {
      "epoch": 0.98,
      "grad_norm": 24.322763442993164,
      "kl": 0.0,
      "learning_rate": 1.1735419630156473e-08,
      "logps/chosen": -313.05377197265625,
      "logps/rejected": -380.57952880859375,
      "loss": 0.1281,
      "rewards/chosen": 2.0147297382354736,
      "rewards/margins": 10.899151802062988,
      "rewards/rejected": -8.884422302246094,
      "step": 1530
    },
    {
      "epoch": 0.98,
      "grad_norm": 15.227581977844238,
      "kl": 0.0,
      "learning_rate": 9.95732574679943e-09,
      "logps/chosen": -231.4268798828125,
      "logps/rejected": -411.5580139160156,
      "loss": 0.1128,
      "rewards/chosen": 2.0990829467773438,
      "rewards/margins": 11.027290344238281,
      "rewards/rejected": -8.928207397460938,
      "step": 1535
    },
    {
      "epoch": 0.99,
      "grad_norm": 21.88052749633789,
      "kl": 0.0,
      "learning_rate": 8.179231863442388e-09,
      "logps/chosen": -233.326416015625,
      "logps/rejected": -443.15234375,
      "loss": 0.1306,
      "rewards/chosen": 1.9721084833145142,
      "rewards/margins": 11.35097885131836,
      "rewards/rejected": -9.378870010375977,
      "step": 1540
    },
    {
      "epoch": 0.99,
      "grad_norm": 17.996585845947266,
      "kl": 0.0,
      "learning_rate": 6.401137980085348e-09,
      "logps/chosen": -264.0694274902344,
      "logps/rejected": -443.869384765625,
      "loss": 0.1047,
      "rewards/chosen": 2.1066718101501465,
      "rewards/margins": 11.750232696533203,
      "rewards/rejected": -9.643560409545898,
      "step": 1545
    },
    {
      "epoch": 0.99,
      "grad_norm": 14.86182689666748,
      "kl": 0.0,
      "learning_rate": 4.623044096728307e-09,
      "logps/chosen": -302.3523864746094,
      "logps/rejected": -436.1973571777344,
      "loss": 0.1027,
      "rewards/chosen": 2.3409790992736816,
      "rewards/margins": 12.13941764831543,
      "rewards/rejected": -9.79843807220459,
      "step": 1550
    },
    {
      "epoch": 0.99,
      "grad_norm": 20.058828353881836,
      "kl": 0.0,
      "learning_rate": 2.844950213371266e-09,
      "logps/chosen": -264.7875061035156,
      "logps/rejected": -401.29241943359375,
      "loss": 0.118,
      "rewards/chosen": 2.1413722038269043,
      "rewards/margins": 11.171432495117188,
      "rewards/rejected": -9.030058860778809,
      "step": 1555
    },
    {
      "epoch": 1.0,
      "grad_norm": 14.352324485778809,
      "kl": 0.0,
      "learning_rate": 1.0668563300142248e-09,
      "logps/chosen": -248.8438262939453,
      "logps/rejected": -420.352783203125,
      "loss": 0.1116,
      "rewards/chosen": 2.209176540374756,
      "rewards/margins": 12.117198944091797,
      "rewards/rejected": -9.9080228805542,
      "step": 1560
    },
    {
      "epoch": 1.0,
      "step": 1563,
      "total_flos": 0.0,
      "train_loss": 0.1710020688315347,
      "train_runtime": 11106.5263,
      "train_samples_per_second": 9.004,
      "train_steps_per_second": 0.141
    }
  ],
  "logging_steps": 5,
  "max_steps": 1563,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}