{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.2616431187859759,
  "eval_steps": 500,
  "global_step": 500,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 25.513092629771464,
      "kl": 0.0,
      "learning_rate": 4.99738356881214e-07,
      "logps/chosen": -255.54945373535156,
      "logps/rejected": -223.1166229248047,
      "loss": 0.2812,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.0,
      "grad_norm": 19.05975346317889,
      "kl": 0.0,
      "learning_rate": 4.99476713762428e-07,
      "logps/chosen": -284.558837890625,
      "logps/rejected": -214.7106475830078,
      "loss": 0.2015,
      "rewards/chosen": 0.34634318947792053,
      "rewards/margins": 0.3400447368621826,
      "rewards/rejected": 0.006298447493463755,
      "step": 2
    },
    {
      "epoch": 0.0,
      "grad_norm": 16.41510277772972,
      "kl": 2.259248733520508,
      "learning_rate": 4.992150706436421e-07,
      "logps/chosen": -191.8242645263672,
      "logps/rejected": -164.48486328125,
      "loss": 0.2416,
      "rewards/chosen": 0.3580303192138672,
      "rewards/margins": 0.17033512890338898,
      "rewards/rejected": 0.1876951903104782,
      "step": 3
    },
    {
      "epoch": 0.0,
      "grad_norm": 18.59921957448892,
      "kl": 1.6011241674423218,
      "learning_rate": 4.989534275248561e-07,
      "logps/chosen": -191.90174865722656,
      "logps/rejected": -228.51425170898438,
      "loss": 0.2344,
      "rewards/chosen": 0.14850404858589172,
      "rewards/margins": 0.18605005741119385,
      "rewards/rejected": -0.03754601255059242,
      "step": 4
    },
    {
      "epoch": 0.0,
      "grad_norm": 19.28790409947415,
      "kl": 0.6965503692626953,
      "learning_rate": 4.986917844060701e-07,
      "logps/chosen": -213.99278259277344,
      "logps/rejected": -244.74432373046875,
      "loss": 0.2692,
      "rewards/chosen": 0.08441495150327682,
      "rewards/margins": -0.010940708220005035,
      "rewards/rejected": 0.09535565972328186,
      "step": 5
    },
    {
      "epoch": 0.0,
      "grad_norm": 16.851732715209323,
      "kl": 0.0,
      "learning_rate": 4.984301412872841e-07,
      "logps/chosen": -235.20309448242188,
      "logps/rejected": -261.23590087890625,
      "loss": 0.1815,
      "rewards/chosen": 0.035614676773548126,
      "rewards/margins": 0.42004701495170593,
      "rewards/rejected": -0.3844323456287384,
      "step": 6
    },
    {
      "epoch": 0.0,
      "grad_norm": 16.58819557091471,
      "kl": 0.0,
      "learning_rate": 4.981684981684982e-07,
      "logps/chosen": -193.42202758789062,
      "logps/rejected": -225.8984832763672,
      "loss": 0.1914,
      "rewards/chosen": -0.15666261315345764,
      "rewards/margins": 0.47593703866004944,
      "rewards/rejected": -0.6325996518135071,
      "step": 7
    },
    {
      "epoch": 0.0,
      "grad_norm": 15.607759793866348,
      "kl": 0.0,
      "learning_rate": 4.979068550497122e-07,
      "logps/chosen": -246.3267059326172,
      "logps/rejected": -197.67164611816406,
      "loss": 0.1615,
      "rewards/chosen": 0.23949605226516724,
      "rewards/margins": 0.5585669279098511,
      "rewards/rejected": -0.3190709054470062,
      "step": 8
    },
    {
      "epoch": 0.0,
      "grad_norm": 17.644608030177945,
      "kl": 0.0,
      "learning_rate": 4.976452119309262e-07,
      "logps/chosen": -270.65924072265625,
      "logps/rejected": -205.1195068359375,
      "loss": 0.2314,
      "rewards/chosen": -0.07605857402086258,
      "rewards/margins": 0.15047526359558105,
      "rewards/rejected": -0.22653384506702423,
      "step": 9
    },
    {
      "epoch": 0.01,
      "grad_norm": 16.618428473287977,
      "kl": 0.0,
      "learning_rate": 4.973835688121403e-07,
      "logps/chosen": -174.2410888671875,
      "logps/rejected": -181.774169921875,
      "loss": 0.2923,
      "rewards/chosen": -0.08562064915895462,
      "rewards/margins": 0.02823999524116516,
      "rewards/rejected": -0.11386064440011978,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 17.243866067758734,
      "kl": 0.0,
      "learning_rate": 4.971219256933543e-07,
      "logps/chosen": -198.24940490722656,
      "logps/rejected": -215.78672790527344,
      "loss": 0.2918,
      "rewards/chosen": 0.13005053997039795,
      "rewards/margins": 0.43379873037338257,
      "rewards/rejected": -0.3037481904029846,
      "step": 11
    },
    {
      "epoch": 0.01,
      "grad_norm": 15.745968199362501,
      "kl": 0.0,
      "learning_rate": 4.968602825745683e-07,
      "logps/chosen": -205.6284637451172,
      "logps/rejected": -214.56349182128906,
      "loss": 0.2833,
      "rewards/chosen": -0.3640637993812561,
      "rewards/margins": 0.07524752616882324,
      "rewards/rejected": -0.43931132555007935,
      "step": 12
    },
    {
      "epoch": 0.01,
      "grad_norm": 15.030324414091437,
      "kl": 0.2821946144104004,
      "learning_rate": 4.965986394557823e-07,
      "logps/chosen": -286.0364990234375,
      "logps/rejected": -206.61985778808594,
      "loss": 0.1812,
      "rewards/chosen": 0.5389153361320496,
      "rewards/margins": 0.35903000831604004,
      "rewards/rejected": 0.17988532781600952,
      "step": 13
    },
    {
      "epoch": 0.01,
      "grad_norm": 16.769570066708727,
      "kl": 0.0,
      "learning_rate": 4.963369963369964e-07,
      "logps/chosen": -198.4276580810547,
      "logps/rejected": -259.9801940917969,
      "loss": 0.2128,
      "rewards/chosen": 0.21227878332138062,
      "rewards/margins": 0.3703112006187439,
      "rewards/rejected": -0.1580324023962021,
      "step": 14
    },
    {
      "epoch": 0.01,
      "grad_norm": 17.041145148374916,
      "kl": 1.2743568420410156,
      "learning_rate": 4.960753532182104e-07,
      "logps/chosen": -229.1648712158203,
      "logps/rejected": -253.59913635253906,
      "loss": 0.2295,
      "rewards/chosen": 0.6795628070831299,
      "rewards/margins": 0.9129478335380554,
      "rewards/rejected": -0.23338502645492554,
      "step": 15
    },
    {
      "epoch": 0.01,
      "grad_norm": 15.28869972325163,
      "kl": 0.0,
      "learning_rate": 4.958137100994244e-07,
      "logps/chosen": -187.63201904296875,
      "logps/rejected": -241.87457275390625,
      "loss": 0.1656,
      "rewards/chosen": 0.1542114019393921,
      "rewards/margins": 0.32588493824005127,
      "rewards/rejected": -0.17167352139949799,
      "step": 16
    },
    {
      "epoch": 0.01,
      "grad_norm": 16.352437946436485,
      "kl": 0.0,
      "learning_rate": 4.955520669806384e-07,
      "logps/chosen": -285.8620910644531,
      "logps/rejected": -275.36328125,
      "loss": 0.229,
      "rewards/chosen": 0.02602279745042324,
      "rewards/margins": 0.5908135175704956,
      "rewards/rejected": -0.5647907257080078,
      "step": 17
    },
    {
      "epoch": 0.01,
      "grad_norm": 14.496172300986403,
      "kl": 0.0,
      "learning_rate": 4.952904238618525e-07,
      "logps/chosen": -236.38160705566406,
      "logps/rejected": -206.48521423339844,
      "loss": 0.127,
      "rewards/chosen": 0.3640650808811188,
      "rewards/margins": 1.3152188062667847,
      "rewards/rejected": -0.9511536955833435,
      "step": 18
    },
    {
      "epoch": 0.01,
      "grad_norm": 15.755801807248107,
      "kl": 0.0,
      "learning_rate": 4.950287807430665e-07,
      "logps/chosen": -261.5793151855469,
      "logps/rejected": -213.00888061523438,
      "loss": 0.2456,
      "rewards/chosen": 0.2305096834897995,
      "rewards/margins": 1.4204968214035034,
      "rewards/rejected": -1.1899871826171875,
      "step": 19
    },
    {
      "epoch": 0.01,
      "grad_norm": 14.150683957656698,
      "kl": 0.0,
      "learning_rate": 4.947671376242805e-07,
      "logps/chosen": -250.30348205566406,
      "logps/rejected": -237.50193786621094,
      "loss": 0.184,
      "rewards/chosen": -0.09291169792413712,
      "rewards/margins": 0.669346809387207,
      "rewards/rejected": -0.7622585296630859,
      "step": 20
    },
    {
      "epoch": 0.01,
      "grad_norm": 16.71784084154209,
      "kl": 0.0,
      "learning_rate": 4.945054945054945e-07,
      "logps/chosen": -250.10418701171875,
      "logps/rejected": -258.0350341796875,
      "loss": 0.2064,
      "rewards/chosen": -0.44092392921447754,
      "rewards/margins": 0.4753909707069397,
      "rewards/rejected": -0.9163148999214172,
      "step": 21
    },
    {
      "epoch": 0.01,
      "grad_norm": 14.027074569038245,
      "kl": 0.0,
      "learning_rate": 4.942438513867084e-07,
      "logps/chosen": -238.38027954101562,
      "logps/rejected": -237.553955078125,
      "loss": 0.2034,
      "rewards/chosen": -0.11189079284667969,
      "rewards/margins": 1.4180889129638672,
      "rewards/rejected": -1.5299797058105469,
      "step": 22
    },
    {
      "epoch": 0.01,
      "grad_norm": 13.242349649129327,
      "kl": 0.0,
      "learning_rate": 4.939822082679226e-07,
      "logps/chosen": -211.67172241210938,
      "logps/rejected": -220.88316345214844,
      "loss": 0.1695,
      "rewards/chosen": -0.0019831450190395117,
      "rewards/margins": 1.3524924516677856,
      "rewards/rejected": -1.3544756174087524,
      "step": 23
    },
    {
      "epoch": 0.01,
      "grad_norm": 14.558526366105085,
      "kl": 0.0,
      "learning_rate": 4.937205651491365e-07,
      "logps/chosen": -183.22792053222656,
      "logps/rejected": -288.7313537597656,
      "loss": 0.3057,
      "rewards/chosen": -0.08994026482105255,
      "rewards/margins": 1.0271720886230469,
      "rewards/rejected": -1.117112398147583,
      "step": 24
    },
    {
      "epoch": 0.01,
      "grad_norm": 14.612720259085432,
      "kl": 0.0,
      "learning_rate": 4.934589220303505e-07,
      "logps/chosen": -183.35086059570312,
      "logps/rejected": -287.0506896972656,
      "loss": 0.1631,
      "rewards/chosen": -0.2845403850078583,
      "rewards/margins": 0.9921867847442627,
      "rewards/rejected": -1.2767271995544434,
      "step": 25
    },
    {
      "epoch": 0.01,
      "grad_norm": 14.1871906272162,
      "kl": 0.0,
      "learning_rate": 4.931972789115645e-07,
      "logps/chosen": -260.46820068359375,
      "logps/rejected": -231.28530883789062,
      "loss": 0.1819,
      "rewards/chosen": -0.6221957802772522,
      "rewards/margins": 0.9863255620002747,
      "rewards/rejected": -1.6085213422775269,
      "step": 26
    },
    {
      "epoch": 0.01,
      "grad_norm": 12.28801127212002,
      "kl": 0.0,
      "learning_rate": 4.929356357927786e-07,
      "logps/chosen": -218.0127410888672,
      "logps/rejected": -236.330078125,
      "loss": 0.2623,
      "rewards/chosen": -1.2400844097137451,
      "rewards/margins": 1.2188379764556885,
      "rewards/rejected": -2.4589223861694336,
      "step": 27
    },
    {
      "epoch": 0.01,
      "grad_norm": 13.969382177172214,
      "kl": 0.0,
      "learning_rate": 4.926739926739926e-07,
      "logps/chosen": -262.0902099609375,
      "logps/rejected": -233.80523681640625,
      "loss": 0.2037,
      "rewards/chosen": -1.2043147087097168,
      "rewards/margins": 1.4312412738800049,
      "rewards/rejected": -2.6355559825897217,
      "step": 28
    },
    {
      "epoch": 0.02,
      "grad_norm": 14.346116577373035,
      "kl": 0.0,
      "learning_rate": 4.924123495552066e-07,
      "logps/chosen": -192.3516387939453,
      "logps/rejected": -301.5596923828125,
      "loss": 0.2649,
      "rewards/chosen": -0.6799588799476624,
      "rewards/margins": 0.6100900769233704,
      "rewards/rejected": -1.2900489568710327,
      "step": 29
    },
    {
      "epoch": 0.02,
      "grad_norm": 14.447170251019777,
      "kl": 0.0,
      "learning_rate": 4.921507064364207e-07,
      "logps/chosen": -246.61720275878906,
      "logps/rejected": -279.7056884765625,
      "loss": 0.1786,
      "rewards/chosen": -0.16088435053825378,
      "rewards/margins": 2.5958094596862793,
      "rewards/rejected": -2.7566938400268555,
      "step": 30
    },
    {
      "epoch": 0.02,
      "grad_norm": 11.638201730563553,
      "kl": 0.0,
      "learning_rate": 4.918890633176347e-07,
      "logps/chosen": -269.7441101074219,
      "logps/rejected": -227.7078094482422,
      "loss": 0.1939,
      "rewards/chosen": 0.17156387865543365,
      "rewards/margins": 2.2280192375183105,
      "rewards/rejected": -2.056455373764038,
      "step": 31
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.742789811988482,
      "kl": 0.0,
      "learning_rate": 4.916274201988487e-07,
      "logps/chosen": -276.08001708984375,
      "logps/rejected": -285.455078125,
      "loss": 0.1504,
      "rewards/chosen": -0.9163249135017395,
      "rewards/margins": 1.5270984172821045,
      "rewards/rejected": -2.443423271179199,
      "step": 32
    },
    {
      "epoch": 0.02,
      "grad_norm": 11.788909838526948,
      "kl": 0.0,
      "learning_rate": 4.913657770800627e-07,
      "logps/chosen": -205.1074981689453,
      "logps/rejected": -295.74462890625,
      "loss": 0.225,
      "rewards/chosen": -0.3685793876647949,
      "rewards/margins": 1.5645514726638794,
      "rewards/rejected": -1.9331308603286743,
      "step": 33
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.82701185590145,
      "kl": 0.0,
      "learning_rate": 4.911041339612768e-07,
      "logps/chosen": -215.7355499267578,
      "logps/rejected": -270.3883056640625,
      "loss": 0.1812,
      "rewards/chosen": -0.46088603138923645,
      "rewards/margins": 1.8914861679077148,
      "rewards/rejected": -2.352372169494629,
      "step": 34
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.279622843857423,
      "kl": 0.0,
      "learning_rate": 4.908424908424908e-07,
      "logps/chosen": -229.2891387939453,
      "logps/rejected": -190.41712951660156,
      "loss": 0.2008,
      "rewards/chosen": 0.11880671977996826,
      "rewards/margins": 1.337594985961914,
      "rewards/rejected": -1.2187882661819458,
      "step": 35
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.833830578324568,
      "kl": 0.0,
      "learning_rate": 4.905808477237048e-07,
      "logps/chosen": -265.9535827636719,
      "logps/rejected": -229.44387817382812,
      "loss": 0.2017,
      "rewards/chosen": 0.7549086213111877,
      "rewards/margins": 2.0256097316741943,
      "rewards/rejected": -1.2707011699676514,
      "step": 36
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.878415132245943,
      "kl": 0.0,
      "learning_rate": 4.903192046049188e-07,
      "logps/chosen": -223.6148681640625,
      "logps/rejected": -232.38751220703125,
      "loss": 0.1578,
      "rewards/chosen": 0.42322105169296265,
      "rewards/margins": 1.5377566814422607,
      "rewards/rejected": -1.1145355701446533,
      "step": 37
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.016348202428658,
      "kl": 0.0,
      "learning_rate": 4.900575614861329e-07,
      "logps/chosen": -221.3059844970703,
      "logps/rejected": -233.40020751953125,
      "loss": 0.153,
      "rewards/chosen": 0.6136862635612488,
      "rewards/margins": 2.5119822025299072,
      "rewards/rejected": -1.8982958793640137,
      "step": 38
    },
    {
      "epoch": 0.02,
      "grad_norm": 11.696306326828102,
      "kl": 0.0,
      "learning_rate": 4.897959183673469e-07,
      "logps/chosen": -263.5332336425781,
      "logps/rejected": -223.24720764160156,
      "loss": 0.2326,
      "rewards/chosen": -0.20513269305229187,
      "rewards/margins": 2.1170685291290283,
      "rewards/rejected": -2.3222012519836426,
      "step": 39
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.797741214492445,
      "kl": 0.0,
      "learning_rate": 4.895342752485609e-07,
      "logps/chosen": -241.70838928222656,
      "logps/rejected": -242.095703125,
      "loss": 0.2954,
      "rewards/chosen": -0.5702841281890869,
      "rewards/margins": 0.34226876497268677,
      "rewards/rejected": -0.9125528931617737,
      "step": 40
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.245463459701265,
      "kl": 0.0,
      "learning_rate": 4.89272632129775e-07,
      "logps/chosen": -200.02525329589844,
      "logps/rejected": -240.9892578125,
      "loss": 0.2966,
      "rewards/chosen": -1.1106632947921753,
      "rewards/margins": -0.09493815898895264,
      "rewards/rejected": -1.0157251358032227,
      "step": 41
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.419341321902724,
      "kl": 0.0,
      "learning_rate": 4.890109890109889e-07,
      "logps/chosen": -224.20755004882812,
      "logps/rejected": -281.16717529296875,
      "loss": 0.1364,
      "rewards/chosen": -0.4932700991630554,
      "rewards/margins": 1.310615062713623,
      "rewards/rejected": -1.8038851022720337,
      "step": 42
    },
    {
      "epoch": 0.02,
      "grad_norm": 11.043427588426145,
      "kl": 0.0,
      "learning_rate": 4.88749345892203e-07,
      "logps/chosen": -233.59933471679688,
      "logps/rejected": -260.2566833496094,
      "loss": 0.2067,
      "rewards/chosen": -0.8917900323867798,
      "rewards/margins": 1.0747038125991821,
      "rewards/rejected": -1.966493844985962,
      "step": 43
    },
    {
      "epoch": 0.02,
      "grad_norm": 12.44162754838529,
      "kl": 0.0,
      "learning_rate": 4.88487702773417e-07,
      "logps/chosen": -216.97845458984375,
      "logps/rejected": -269.1575012207031,
      "loss": 0.2059,
      "rewards/chosen": 0.21655617654323578,
      "rewards/margins": 2.175200939178467,
      "rewards/rejected": -1.9586448669433594,
      "step": 44
    },
    {
      "epoch": 0.02,
      "grad_norm": 13.803513332755253,
      "kl": 0.0,
      "learning_rate": 4.88226059654631e-07,
      "logps/chosen": -265.6169128417969,
      "logps/rejected": -308.48760986328125,
      "loss": 0.1974,
      "rewards/chosen": -0.7583664059638977,
      "rewards/margins": 1.8766412734985352,
      "rewards/rejected": -2.635007619857788,
      "step": 45
    },
    {
      "epoch": 0.02,
      "grad_norm": 13.259849756023506,
      "kl": 0.0,
      "learning_rate": 4.879644165358451e-07,
      "logps/chosen": -216.37884521484375,
      "logps/rejected": -263.8242492675781,
      "loss": 0.1924,
      "rewards/chosen": -0.5627312660217285,
      "rewards/margins": 1.7108609676361084,
      "rewards/rejected": -2.273592233657837,
      "step": 46
    },
    {
      "epoch": 0.02,
      "grad_norm": 11.401346938362929,
      "kl": 0.0,
      "learning_rate": 4.877027734170591e-07,
      "logps/chosen": -196.87896728515625,
      "logps/rejected": -168.52236938476562,
      "loss": 0.3164,
      "rewards/chosen": -0.26614972949028015,
      "rewards/margins": 1.1080857515335083,
      "rewards/rejected": -1.3742355108261108,
      "step": 47
    },
    {
      "epoch": 0.03,
      "grad_norm": 10.108076253718023,
      "kl": 0.0,
      "learning_rate": 4.874411302982731e-07,
      "logps/chosen": -207.7807159423828,
      "logps/rejected": -219.1448974609375,
      "loss": 0.1305,
      "rewards/chosen": -1.0533264875411987,
      "rewards/margins": 1.3320115804672241,
      "rewards/rejected": -2.385338068008423,
      "step": 48
    },
    {
      "epoch": 0.03,
      "grad_norm": 10.762036932864307,
      "kl": 0.0,
      "learning_rate": 4.871794871794871e-07,
      "logps/chosen": -236.81471252441406,
      "logps/rejected": -228.0085906982422,
      "loss": 0.1264,
      "rewards/chosen": 0.9120302796363831,
      "rewards/margins": 2.634833335876465,
      "rewards/rejected": -1.722802996635437,
      "step": 49
    },
    {
      "epoch": 0.03,
      "grad_norm": 11.066959845009587,
      "kl": 0.0,
      "learning_rate": 4.869178440607012e-07,
      "logps/chosen": -252.11825561523438,
      "logps/rejected": -209.72384643554688,
      "loss": 0.2249,
      "rewards/chosen": -0.5552583336830139,
      "rewards/margins": 1.0669043064117432,
      "rewards/rejected": -1.6221626996994019,
      "step": 50
    },
    {
      "epoch": 0.03,
      "grad_norm": 11.898163000668239,
      "kl": 0.0,
      "learning_rate": 4.866562009419152e-07,
      "logps/chosen": -229.1280975341797,
      "logps/rejected": -239.55064392089844,
      "loss": 0.2366,
      "rewards/chosen": -0.3932199478149414,
      "rewards/margins": 0.8869756460189819,
      "rewards/rejected": -1.2801955938339233,
      "step": 51
    },
    {
      "epoch": 0.03,
      "grad_norm": 11.53373054095599,
      "kl": 0.0,
      "learning_rate": 4.863945578231292e-07,
      "logps/chosen": -282.8030090332031,
      "logps/rejected": -215.99546813964844,
      "loss": 0.176,
      "rewards/chosen": -0.04715963453054428,
      "rewards/margins": 1.483907699584961,
      "rewards/rejected": -1.5310673713684082,
      "step": 52
    },
    {
      "epoch": 0.03,
      "grad_norm": 10.818173532721506,
      "kl": 0.0,
      "learning_rate": 4.861329147043432e-07,
      "logps/chosen": -170.08155822753906,
      "logps/rejected": -266.9096984863281,
      "loss": 0.196,
      "rewards/chosen": -0.43234577775001526,
      "rewards/margins": 2.175133466720581,
      "rewards/rejected": -2.6074793338775635,
      "step": 53
    },
    {
      "epoch": 0.03,
      "grad_norm": 12.389896501721763,
      "kl": 0.0,
      "learning_rate": 4.858712715855573e-07,
      "logps/chosen": -188.79185485839844,
      "logps/rejected": -217.9726104736328,
      "loss": 0.1855,
      "rewards/chosen": -0.7043643593788147,
      "rewards/margins": 1.4354064464569092,
      "rewards/rejected": -2.139770746231079,
      "step": 54
    },
    {
      "epoch": 0.03,
      "grad_norm": 10.719323563675456,
      "kl": 0.0,
      "learning_rate": 4.856096284667713e-07,
      "logps/chosen": -235.02947998046875,
      "logps/rejected": -264.5641174316406,
      "loss": 0.217,
      "rewards/chosen": -1.3881831169128418,
      "rewards/margins": 2.237257480621338,
      "rewards/rejected": -3.6254405975341797,
      "step": 55
    },
    {
      "epoch": 0.03,
      "grad_norm": 11.632198403788388,
      "kl": 0.0,
      "learning_rate": 4.853479853479853e-07,
      "logps/chosen": -281.6618347167969,
      "logps/rejected": -228.85995483398438,
      "loss": 0.0917,
      "rewards/chosen": 0.013584318570792675,
      "rewards/margins": 3.580209732055664,
      "rewards/rejected": -3.5666253566741943,
      "step": 56
    },
    {
      "epoch": 0.03,
      "grad_norm": 12.518341857853112,
      "kl": 0.0,
      "learning_rate": 4.850863422291994e-07,
      "logps/chosen": -159.27957153320312,
      "logps/rejected": -251.3924560546875,
      "loss": 0.21,
      "rewards/chosen": -0.8034593462944031,
      "rewards/margins": 1.3652153015136719,
      "rewards/rejected": -2.1686747074127197,
      "step": 57
    },
    {
      "epoch": 0.03,
      "grad_norm": 11.773486848179154,
      "kl": 0.0,
      "learning_rate": 4.848246991104133e-07,
      "logps/chosen": -163.08563232421875,
      "logps/rejected": -225.0583038330078,
      "loss": 0.2323,
      "rewards/chosen": -0.784336268901825,
      "rewards/margins": 0.35770100355148315,
      "rewards/rejected": -1.142037272453308,
      "step": 58
    },
    {
      "epoch": 0.03,
      "grad_norm": 9.98418104323235,
      "kl": 0.0,
      "learning_rate": 4.845630559916274e-07,
      "logps/chosen": -250.50967407226562,
      "logps/rejected": -260.6304016113281,
      "loss": 0.1682,
      "rewards/chosen": -0.046240393072366714,
      "rewards/margins": 2.6554722785949707,
      "rewards/rejected": -2.7017126083374023,
      "step": 59
    },
    {
      "epoch": 0.03,
      "grad_norm": 13.45864474945625,
      "kl": 0.0,
      "learning_rate": 4.843014128728414e-07,
      "logps/chosen": -241.01133728027344,
      "logps/rejected": -208.72291564941406,
      "loss": 0.2261,
      "rewards/chosen": -1.1776716709136963,
      "rewards/margins": 0.5967569351196289,
      "rewards/rejected": -1.7744286060333252,
      "step": 60
    },
    {
      "epoch": 0.03,
      "grad_norm": 10.967311785129077,
      "kl": 0.0,
      "learning_rate": 4.840397697540555e-07,
      "logps/chosen": -240.189453125,
      "logps/rejected": -213.64976501464844,
      "loss": 0.2169,
      "rewards/chosen": -0.17561621963977814,
      "rewards/margins": 2.7882721424102783,
      "rewards/rejected": -2.96388840675354,
      "step": 61
    },
    {
      "epoch": 0.03,
      "grad_norm": 10.93191003753912,
      "kl": 0.0,
      "learning_rate": 4.837781266352695e-07,
      "logps/chosen": -227.45742797851562,
      "logps/rejected": -187.86070251464844,
      "loss": 0.1054,
      "rewards/chosen": -0.3846696615219116,
      "rewards/margins": 1.533595085144043,
      "rewards/rejected": -1.9182647466659546,
      "step": 62
    },
    {
      "epoch": 0.03,
      "grad_norm": 9.831720988058269,
      "kl": 0.0,
      "learning_rate": 4.835164835164835e-07,
      "logps/chosen": -273.84375,
      "logps/rejected": -213.56747436523438,
      "loss": 0.1685,
      "rewards/chosen": -0.3652818500995636,
      "rewards/margins": 2.15742564201355,
      "rewards/rejected": -2.522707462310791,
      "step": 63
    },
    {
      "epoch": 0.03,
      "grad_norm": 12.175263730184788,
      "kl": 0.0,
      "learning_rate": 4.832548403976975e-07,
      "logps/chosen": -244.45236206054688,
      "logps/rejected": -183.27474975585938,
      "loss": 0.1439,
      "rewards/chosen": 0.15830610692501068,
      "rewards/margins": 2.1498560905456543,
      "rewards/rejected": -1.9915499687194824,
      "step": 64
    },
    {
      "epoch": 0.03,
      "grad_norm": 12.593612516595744,
      "kl": 0.0,
      "learning_rate": 4.829931972789115e-07,
      "logps/chosen": -265.2001953125,
      "logps/rejected": -295.2158508300781,
      "loss": 0.1956,
      "rewards/chosen": -0.9956868886947632,
      "rewards/margins": 0.9066202640533447,
      "rewards/rejected": -1.902307152748108,
      "step": 65
    },
    {
      "epoch": 0.03,
      "grad_norm": 12.056826564998115,
      "kl": 0.0,
      "learning_rate": 4.827315541601256e-07,
      "logps/chosen": -192.40469360351562,
      "logps/rejected": -247.81382751464844,
      "loss": 0.178,
      "rewards/chosen": -0.7804039120674133,
      "rewards/margins": 1.2094306945800781,
      "rewards/rejected": -1.9898346662521362,
      "step": 66
    },
    {
      "epoch": 0.04,
      "grad_norm": 13.384738493954586,
      "kl": 0.0,
      "learning_rate": 4.824699110413396e-07,
      "logps/chosen": -159.72280883789062,
      "logps/rejected": -328.75616455078125,
      "loss": 0.1926,
      "rewards/chosen": -0.9741663932800293,
      "rewards/margins": 1.3278543949127197,
      "rewards/rejected": -2.302020788192749,
      "step": 67
    },
    {
      "epoch": 0.04,
      "grad_norm": 12.538951694906014,
      "kl": 0.0,
      "learning_rate": 4.822082679225536e-07,
      "logps/chosen": -254.5558319091797,
      "logps/rejected": -190.9063262939453,
      "loss": 0.2038,
      "rewards/chosen": -0.5719714164733887,
      "rewards/margins": 1.5436241626739502,
      "rewards/rejected": -2.115595579147339,
      "step": 68
    },
    {
      "epoch": 0.04,
      "grad_norm": 11.328347418708772,
      "kl": 0.0,
      "learning_rate": 4.819466248037676e-07,
      "logps/chosen": -237.08462524414062,
      "logps/rejected": -318.36474609375,
      "loss": 0.327,
      "rewards/chosen": -1.5885767936706543,
      "rewards/margins": 1.1602427959442139,
      "rewards/rejected": -2.748819589614868,
      "step": 69
    },
    {
      "epoch": 0.04,
      "grad_norm": 12.400662861104186,
      "kl": 0.0,
      "learning_rate": 4.816849816849817e-07,
      "logps/chosen": -238.31106567382812,
      "logps/rejected": -195.43804931640625,
      "loss": 0.1737,
      "rewards/chosen": -0.6987349987030029,
      "rewards/margins": 2.256924629211426,
      "rewards/rejected": -2.9556596279144287,
      "step": 70
    },
    {
      "epoch": 0.04,
      "grad_norm": 12.841610853219839,
      "kl": 0.0,
      "learning_rate": 4.814233385661957e-07,
      "logps/chosen": -204.5384521484375,
      "logps/rejected": -208.86331176757812,
      "loss": 0.2565,
      "rewards/chosen": -1.2488123178482056,
      "rewards/margins": 1.1284335851669312,
      "rewards/rejected": -2.3772459030151367,
      "step": 71
    },
    {
      "epoch": 0.04,
      "grad_norm": 13.054533599473265,
      "kl": 0.0,
      "learning_rate": 4.811616954474097e-07,
      "logps/chosen": -250.12615966796875,
      "logps/rejected": -243.621337890625,
      "loss": 0.1785,
      "rewards/chosen": -0.7167716026306152,
      "rewards/margins": 1.7554688453674316,
      "rewards/rejected": -2.472240447998047,
      "step": 72
    },
    {
      "epoch": 0.04,
      "grad_norm": 11.218204777591135,
      "kl": 0.0,
      "learning_rate": 4.809000523286237e-07,
      "logps/chosen": -202.5470733642578,
      "logps/rejected": -251.5446014404297,
      "loss": 0.2578,
      "rewards/chosen": -0.35193824768066406,
      "rewards/margins": 1.2476764917373657,
      "rewards/rejected": -1.5996147394180298,
      "step": 73
    },
    {
      "epoch": 0.04,
      "grad_norm": 10.005076969894088,
      "kl": 0.0,
      "learning_rate": 4.806384092098378e-07,
      "logps/chosen": -240.54025268554688,
      "logps/rejected": -238.83575439453125,
      "loss": 0.2025,
      "rewards/chosen": -1.1610413789749146,
      "rewards/margins": 1.6202930212020874,
      "rewards/rejected": -2.781334400177002,
      "step": 74
    },
    {
      "epoch": 0.04,
      "grad_norm": 13.421231936678494,
      "kl": 0.0,
      "learning_rate": 4.803767660910518e-07,
      "logps/chosen": -287.82110595703125,
      "logps/rejected": -193.2252960205078,
      "loss": 0.2056,
      "rewards/chosen": -0.41320356726646423,
      "rewards/margins": 2.6422228813171387,
      "rewards/rejected": -3.0554263591766357,
      "step": 75
    },
    {
      "epoch": 0.04,
      "grad_norm": 11.604997350152324,
      "kl": 0.0,
      "learning_rate": 4.801151229722658e-07,
      "logps/chosen": -266.6557922363281,
      "logps/rejected": -204.9314727783203,
      "loss": 0.1635,
      "rewards/chosen": -0.5626986622810364,
      "rewards/margins": 1.7505519390106201,
      "rewards/rejected": -2.3132505416870117,
      "step": 76
    },
    {
      "epoch": 0.04,
      "grad_norm": 13.167949690406468,
      "kl": 0.0,
      "learning_rate": 4.798534798534799e-07,
      "logps/chosen": -212.5312957763672,
      "logps/rejected": -245.55245971679688,
      "loss": 0.2015,
      "rewards/chosen": -0.6334313750267029,
      "rewards/margins": 0.9760074019432068,
      "rewards/rejected": -1.6094387769699097,
      "step": 77
    },
    {
      "epoch": 0.04,
      "grad_norm": 13.71229302802406,
      "kl": 0.0,
      "learning_rate": 4.795918367346938e-07,
      "logps/chosen": -248.7874755859375,
      "logps/rejected": -254.36085510253906,
      "loss": 0.2529,
      "rewards/chosen": 0.024569960311055183,
      "rewards/margins": 1.5464191436767578,
      "rewards/rejected": -1.5218491554260254,
      "step": 78
    },
    {
      "epoch": 0.04,
      "grad_norm": 12.206901910051194,
      "kl": 0.0,
      "learning_rate": 4.793301936159079e-07,
      "logps/chosen": -214.5361328125,
      "logps/rejected": -224.99876403808594,
      "loss": 0.1976,
      "rewards/chosen": -0.3784463107585907,
      "rewards/margins": 2.108283281326294,
      "rewards/rejected": -2.486729621887207,
      "step": 79
    },
    {
      "epoch": 0.04,
      "grad_norm": 12.358475151298732,
      "kl": 0.0,
      "learning_rate": 4.790685504971219e-07,
      "logps/chosen": -227.8341064453125,
      "logps/rejected": -244.53817749023438,
      "loss": 0.1508,
      "rewards/chosen": 0.0641309916973114,
      "rewards/margins": 2.4745709896087646,
      "rewards/rejected": -2.410439968109131,
      "step": 80
    },
    {
      "epoch": 0.04,
      "grad_norm": 10.7530961108445,
      "kl": 0.0,
      "learning_rate": 4.78806907378336e-07,
      "logps/chosen": -197.07553100585938,
      "logps/rejected": -282.6841735839844,
      "loss": 0.2055,
      "rewards/chosen": -1.8003284931182861,
      "rewards/margins": 0.7180378437042236,
      "rewards/rejected": -2.5183663368225098,
      "step": 81
    },
    {
      "epoch": 0.04,
      "grad_norm": 15.543960456447536,
      "kl": 0.0,
      "learning_rate": 4.7854526425955e-07,
      "logps/chosen": -286.0069580078125,
      "logps/rejected": -333.1952819824219,
      "loss": 0.2215,
      "rewards/chosen": -0.9929041862487793,
      "rewards/margins": 1.838501214981079,
      "rewards/rejected": -2.8314054012298584,
      "step": 82
    },
    {
      "epoch": 0.04,
      "grad_norm": 12.540856467590372,
      "kl": 0.0,
      "learning_rate": 4.78283621140764e-07,
      "logps/chosen": -250.1410369873047,
      "logps/rejected": -234.04039001464844,
      "loss": 0.2142,
      "rewards/chosen": -1.1462880373001099,
      "rewards/margins": 1.3604058027267456,
      "rewards/rejected": -2.5066938400268555,
      "step": 83
    },
    {
      "epoch": 0.04,
      "grad_norm": 12.685162706946018,
      "kl": 0.0,
      "learning_rate": 4.78021978021978e-07,
      "logps/chosen": -244.63720703125,
      "logps/rejected": -206.22079467773438,
      "loss": 0.2197,
      "rewards/chosen": -0.5177630186080933,
      "rewards/margins": 2.0680246353149414,
      "rewards/rejected": -2.585787534713745,
      "step": 84
    },
    {
      "epoch": 0.04,
      "grad_norm": 10.152423089357503,
      "kl": 0.0,
      "learning_rate": 4.77760334903192e-07,
      "logps/chosen": -208.19064331054688,
      "logps/rejected": -250.68597412109375,
      "loss": 0.0981,
      "rewards/chosen": 0.4294729232788086,
      "rewards/margins": 2.881974697113037,
      "rewards/rejected": -2.4525017738342285,
      "step": 85
    },
    {
      "epoch": 0.05,
      "grad_norm": 10.497367332273754,
      "kl": 0.0,
      "learning_rate": 4.774986917844061e-07,
      "logps/chosen": -200.9931640625,
      "logps/rejected": -224.72264099121094,
      "loss": 0.2253,
      "rewards/chosen": -0.9502390623092651,
      "rewards/margins": 0.7048424482345581,
      "rewards/rejected": -1.6550815105438232,
      "step": 86
    },
    {
      "epoch": 0.05,
      "grad_norm": 10.454551429296167,
      "kl": 0.0,
      "learning_rate": 4.772370486656201e-07,
      "logps/chosen": -204.69024658203125,
      "logps/rejected": -238.31492614746094,
      "loss": 0.1596,
      "rewards/chosen": 0.6356328725814819,
      "rewards/margins": 2.17765474319458,
      "rewards/rejected": -1.5420219898223877,
      "step": 87
    },
    {
      "epoch": 0.05,
      "grad_norm": 11.947921924106597,
      "kl": 0.0,
      "learning_rate": 4.769754055468341e-07,
      "logps/chosen": -233.5601806640625,
      "logps/rejected": -232.72801208496094,
      "loss": 0.2242,
      "rewards/chosen": 0.4331187307834625,
      "rewards/margins": 1.7527743577957153,
      "rewards/rejected": -1.3196556568145752,
      "step": 88
    },
    {
      "epoch": 0.05,
      "grad_norm": 12.796884542926785,
      "kl": 0.0,
      "learning_rate": 4.767137624280481e-07,
      "logps/chosen": -229.7658233642578,
      "logps/rejected": -266.162841796875,
      "loss": 0.2534,
      "rewards/chosen": -0.2590081989765167,
      "rewards/margins": 1.418634295463562,
      "rewards/rejected": -1.6776424646377563,
      "step": 89
    },
    {
      "epoch": 0.05,
      "grad_norm": 10.31450194916881,
      "kl": 0.0,
      "learning_rate": 4.764521193092622e-07,
      "logps/chosen": -273.9610595703125,
      "logps/rejected": -277.6679382324219,
      "loss": 0.1663,
      "rewards/chosen": -0.03371845558285713,
      "rewards/margins": 2.451915740966797,
      "rewards/rejected": -2.4856340885162354,
      "step": 90
    },
    {
      "epoch": 0.05,
      "grad_norm": 9.278455988308327,
      "kl": 0.0,
      "learning_rate": 4.761904761904761e-07,
      "logps/chosen": -232.08615112304688,
      "logps/rejected": -272.5909118652344,
      "loss": 0.184,
      "rewards/chosen": -0.7493877410888672,
      "rewards/margins": 2.300028085708618,
      "rewards/rejected": -3.0494158267974854,
      "step": 91
    },
    {
      "epoch": 0.05,
      "grad_norm": 8.817410071740532,
      "kl": 0.0,
      "learning_rate": 4.7592883307169017e-07,
      "logps/chosen": -222.3486785888672,
      "logps/rejected": -248.85317993164062,
      "loss": 0.1731,
      "rewards/chosen": -0.2696648836135864,
      "rewards/margins": 2.0320448875427246,
      "rewards/rejected": -2.3017096519470215,
      "step": 92
    },
    {
      "epoch": 0.05,
      "grad_norm": 10.59864769694075,
      "kl": 0.0,
      "learning_rate": 4.756671899529042e-07,
      "logps/chosen": -189.18434143066406,
      "logps/rejected": -270.7514343261719,
      "loss": 0.1508,
      "rewards/chosen": 0.2290552258491516,
      "rewards/margins": 3.3581156730651855,
      "rewards/rejected": -3.1290605068206787,
      "step": 93
    },
    {
      "epoch": 0.05,
      "grad_norm": 11.988809402174233,
      "kl": 0.0,
      "learning_rate": 4.7540554683411827e-07,
      "logps/chosen": -276.038330078125,
      "logps/rejected": -303.55596923828125,
      "loss": 0.191,
      "rewards/chosen": -0.4145098030567169,
      "rewards/margins": 2.4649319648742676,
      "rewards/rejected": -2.879441738128662,
      "step": 94
    },
    {
      "epoch": 0.05,
      "grad_norm": 14.079727761013277,
      "kl": 0.0,
      "learning_rate": 4.7514390371533227e-07,
      "logps/chosen": -257.2515869140625,
      "logps/rejected": -244.81591796875,
      "loss": 0.2074,
      "rewards/chosen": -1.449769139289856,
      "rewards/margins": 2.1893882751464844,
      "rewards/rejected": -3.63915753364563,
      "step": 95
    },
    {
      "epoch": 0.05,
      "grad_norm": 12.455104492784045,
      "kl": 0.0,
      "learning_rate": 4.7488226059654627e-07,
      "logps/chosen": -258.3236389160156,
      "logps/rejected": -223.65472412109375,
      "loss": 0.2373,
      "rewards/chosen": -0.474732369184494,
      "rewards/margins": 1.6088600158691406,
      "rewards/rejected": -2.083592414855957,
      "step": 96
    },
    {
      "epoch": 0.05,
      "grad_norm": 12.813735596225209,
      "kl": 0.0,
      "learning_rate": 4.746206174777603e-07,
      "logps/chosen": -250.6949005126953,
      "logps/rejected": -282.6470031738281,
      "loss": 0.2388,
      "rewards/chosen": -0.8905995488166809,
      "rewards/margins": 1.769092321395874,
      "rewards/rejected": -2.65969181060791,
      "step": 97
    },
    {
      "epoch": 0.05,
      "grad_norm": 11.204915138695135,
      "kl": 0.0,
      "learning_rate": 4.743589743589743e-07,
      "logps/chosen": -176.22486877441406,
      "logps/rejected": -309.79229736328125,
      "loss": 0.2097,
      "rewards/chosen": -1.4110625982284546,
      "rewards/margins": 0.993507981300354,
      "rewards/rejected": -2.4045705795288086,
      "step": 98
    },
    {
      "epoch": 0.05,
      "grad_norm": 11.735425350671436,
      "kl": 0.0,
      "learning_rate": 4.7409733124018836e-07,
      "logps/chosen": -224.20626831054688,
      "logps/rejected": -246.65956115722656,
      "loss": 0.1503,
      "rewards/chosen": -0.4009709060192108,
      "rewards/margins": 2.9179859161376953,
      "rewards/rejected": -3.3189568519592285,
      "step": 99
    },
    {
      "epoch": 0.05,
      "grad_norm": 9.89695125192596,
      "kl": 0.0,
      "learning_rate": 4.738356881214024e-07,
      "logps/chosen": -217.57763671875,
      "logps/rejected": -224.3143768310547,
      "loss": 0.1092,
      "rewards/chosen": 0.3369201123714447,
      "rewards/margins": 2.3455708026885986,
      "rewards/rejected": -2.008650779724121,
      "step": 100
    },
    {
      "epoch": 0.05,
      "grad_norm": 10.212358128672067,
      "kl": 0.0,
      "learning_rate": 4.7357404500261636e-07,
      "logps/chosen": -197.8075408935547,
      "logps/rejected": -229.45677185058594,
      "loss": 0.2126,
      "rewards/chosen": -0.7042273879051208,
      "rewards/margins": 1.3926599025726318,
      "rewards/rejected": -2.0968873500823975,
      "step": 101
    },
    {
      "epoch": 0.05,
      "grad_norm": 12.604368131047796,
      "kl": 0.0,
      "learning_rate": 4.733124018838304e-07,
      "logps/chosen": -211.51576232910156,
      "logps/rejected": -228.77308654785156,
      "loss": 0.2363,
      "rewards/chosen": -1.1265863180160522,
      "rewards/margins": 0.9696308374404907,
      "rewards/rejected": -2.096217155456543,
      "step": 102
    },
    {
      "epoch": 0.05,
      "grad_norm": 11.118276337061534,
      "kl": 0.0,
      "learning_rate": 4.7305075876504446e-07,
      "logps/chosen": -199.48486328125,
      "logps/rejected": -370.7545166015625,
      "loss": 0.1847,
      "rewards/chosen": -0.5413498282432556,
      "rewards/margins": 2.3086533546447754,
      "rewards/rejected": -2.850003242492676,
      "step": 103
    },
    {
      "epoch": 0.05,
      "grad_norm": 13.028026388315732,
      "kl": 0.0,
      "learning_rate": 4.727891156462585e-07,
      "logps/chosen": -262.6328430175781,
      "logps/rejected": -287.1793518066406,
      "loss": 0.2251,
      "rewards/chosen": -0.2909429669380188,
      "rewards/margins": 2.515258550643921,
      "rewards/rejected": -2.806201457977295,
      "step": 104
    },
    {
      "epoch": 0.05,
      "grad_norm": 11.421812108784723,
      "kl": 0.0,
      "learning_rate": 4.725274725274725e-07,
      "logps/chosen": -234.4532470703125,
      "logps/rejected": -272.8806457519531,
      "loss": 0.1591,
      "rewards/chosen": 0.45129498839378357,
      "rewards/margins": 2.3014488220214844,
      "rewards/rejected": -1.8501538038253784,
      "step": 105
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.451980811425736,
      "kl": 0.0,
      "learning_rate": 4.7226582940868656e-07,
      "logps/chosen": -266.4130859375,
      "logps/rejected": -256.59039306640625,
      "loss": 0.1315,
      "rewards/chosen": -0.39178353548049927,
      "rewards/margins": 1.9923765659332275,
      "rewards/rejected": -2.384160041809082,
      "step": 106
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.541336180624638,
      "kl": 0.0,
      "learning_rate": 4.7200418628990056e-07,
      "logps/chosen": -253.11045837402344,
      "logps/rejected": -257.3990478515625,
      "loss": 0.1589,
      "rewards/chosen": 0.02405649609863758,
      "rewards/margins": 3.012369394302368,
      "rewards/rejected": -2.9883129596710205,
      "step": 107
    },
    {
      "epoch": 0.06,
      "grad_norm": 11.191607092594259,
      "kl": 0.0,
      "learning_rate": 4.7174254317111455e-07,
      "logps/chosen": -219.4788818359375,
      "logps/rejected": -251.76841735839844,
      "loss": 0.1878,
      "rewards/chosen": -0.653347909450531,
      "rewards/margins": 1.0217313766479492,
      "rewards/rejected": -1.675079345703125,
      "step": 108
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.465911824092663,
      "kl": 0.0,
      "learning_rate": 4.714809000523286e-07,
      "logps/chosen": -253.79640197753906,
      "logps/rejected": -208.65074157714844,
      "loss": 0.2581,
      "rewards/chosen": -1.6879644393920898,
      "rewards/margins": -0.33700597286224365,
      "rewards/rejected": -1.3509584665298462,
      "step": 109
    },
    {
      "epoch": 0.06,
      "grad_norm": 11.000969752280952,
      "kl": 0.0,
      "learning_rate": 4.7121925693354265e-07,
      "logps/chosen": -251.3641815185547,
      "logps/rejected": -240.36642456054688,
      "loss": 0.1976,
      "rewards/chosen": -0.2726837396621704,
      "rewards/margins": 2.477322578430176,
      "rewards/rejected": -2.7500064373016357,
      "step": 110
    },
    {
      "epoch": 0.06,
      "grad_norm": 14.908402118189745,
      "kl": 0.0,
      "learning_rate": 4.7095761381475665e-07,
      "logps/chosen": -244.45858764648438,
      "logps/rejected": -266.1649169921875,
      "loss": 0.1959,
      "rewards/chosen": -0.5489299893379211,
      "rewards/margins": 0.9248166680335999,
      "rewards/rejected": -1.473746657371521,
      "step": 111
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.994589492437557,
      "kl": 0.0,
      "learning_rate": 4.7069597069597065e-07,
      "logps/chosen": -250.3166961669922,
      "logps/rejected": -275.6036071777344,
      "loss": 0.2578,
      "rewards/chosen": -1.3922420740127563,
      "rewards/margins": 0.7932580709457397,
      "rewards/rejected": -2.185500144958496,
      "step": 112
    },
    {
      "epoch": 0.06,
      "grad_norm": 12.230772963233404,
      "kl": 0.0,
      "learning_rate": 4.704343275771847e-07,
      "logps/chosen": -251.1761474609375,
      "logps/rejected": -354.3590393066406,
      "loss": 0.1702,
      "rewards/chosen": 0.4129093289375305,
      "rewards/margins": 1.9088201522827148,
      "rewards/rejected": -1.4959107637405396,
      "step": 113
    },
    {
      "epoch": 0.06,
      "grad_norm": 11.73992280402579,
      "kl": 0.0,
      "learning_rate": 4.7017268445839875e-07,
      "logps/chosen": -185.67906188964844,
      "logps/rejected": -188.59017944335938,
      "loss": 0.1331,
      "rewards/chosen": -0.09853968024253845,
      "rewards/margins": 1.8528465032577515,
      "rewards/rejected": -1.9513862133026123,
      "step": 114
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.142692814638496,
      "kl": 0.0,
      "learning_rate": 4.6991104133961275e-07,
      "logps/chosen": -266.6419982910156,
      "logps/rejected": -270.0727233886719,
      "loss": 0.1801,
      "rewards/chosen": -0.9462317824363708,
      "rewards/margins": 1.412889003753662,
      "rewards/rejected": -2.3591208457946777,
      "step": 115
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.967127165803278,
      "kl": 0.0,
      "learning_rate": 4.696493982208268e-07,
      "logps/chosen": -247.25096130371094,
      "logps/rejected": -212.33914184570312,
      "loss": 0.1917,
      "rewards/chosen": -1.022446632385254,
      "rewards/margins": -0.13641774654388428,
      "rewards/rejected": -0.8860288858413696,
      "step": 116
    },
    {
      "epoch": 0.06,
      "grad_norm": 12.672315042849133,
      "kl": 0.0,
      "learning_rate": 4.693877551020408e-07,
      "logps/chosen": -251.9243621826172,
      "logps/rejected": -278.6723327636719,
      "loss": 0.2143,
      "rewards/chosen": -0.1397348791360855,
      "rewards/margins": 2.0988736152648926,
      "rewards/rejected": -2.2386085987091064,
      "step": 117
    },
    {
      "epoch": 0.06,
      "grad_norm": 9.90707282468987,
      "kl": 0.0,
      "learning_rate": 4.691261119832548e-07,
      "logps/chosen": -198.83712768554688,
      "logps/rejected": -277.9150085449219,
      "loss": 0.241,
      "rewards/chosen": -0.4362916052341461,
      "rewards/margins": 2.6319870948791504,
      "rewards/rejected": -3.0682787895202637,
      "step": 118
    },
    {
      "epoch": 0.06,
      "grad_norm": 9.980080260830226,
      "kl": 0.0,
      "learning_rate": 4.6886446886446884e-07,
      "logps/chosen": -222.55442810058594,
      "logps/rejected": -239.49298095703125,
      "loss": 0.1667,
      "rewards/chosen": -0.26491254568099976,
      "rewards/margins": 2.8245887756347656,
      "rewards/rejected": -3.08950138092041,
      "step": 119
    },
    {
      "epoch": 0.06,
      "grad_norm": 14.223816837655765,
      "kl": 0.0,
      "learning_rate": 4.686028257456829e-07,
      "logps/chosen": -229.81675720214844,
      "logps/rejected": -248.0295867919922,
      "loss": 0.2241,
      "rewards/chosen": 0.6647676229476929,
      "rewards/margins": 2.933067798614502,
      "rewards/rejected": -2.2683000564575195,
      "step": 120
    },
    {
      "epoch": 0.06,
      "grad_norm": 11.062546156999915,
      "kl": 0.0,
      "learning_rate": 4.683411826268969e-07,
      "logps/chosen": -215.87423706054688,
      "logps/rejected": -205.9690399169922,
      "loss": 0.221,
      "rewards/chosen": -0.38256943225860596,
      "rewards/margins": 1.25730562210083,
      "rewards/rejected": -1.639875054359436,
      "step": 121
    },
    {
      "epoch": 0.06,
      "grad_norm": 11.71515711397141,
      "kl": 0.0,
      "learning_rate": 4.680795395081109e-07,
      "logps/chosen": -195.1471710205078,
      "logps/rejected": -204.15228271484375,
      "loss": 0.205,
      "rewards/chosen": 0.031107014045119286,
      "rewards/margins": 1.4260950088500977,
      "rewards/rejected": -1.394987940788269,
      "step": 122
    },
    {
      "epoch": 0.06,
      "grad_norm": 10.668033463026218,
      "kl": 0.0,
      "learning_rate": 4.6781789638932494e-07,
      "logps/chosen": -238.59262084960938,
      "logps/rejected": -244.41387939453125,
      "loss": 0.2863,
      "rewards/chosen": -0.6594109535217285,
      "rewards/margins": 0.7834258079528809,
      "rewards/rejected": -1.4428367614746094,
      "step": 123
    },
    {
      "epoch": 0.06,
      "grad_norm": 13.050812790862148,
      "kl": 0.0,
      "learning_rate": 4.6755625327053893e-07,
      "logps/chosen": -208.3390655517578,
      "logps/rejected": -246.9788360595703,
      "loss": 0.1784,
      "rewards/chosen": -0.5635675191879272,
      "rewards/margins": 0.9708378314971924,
      "rewards/rejected": -1.5344053506851196,
      "step": 124
    },
    {
      "epoch": 0.07,
      "grad_norm": 11.38382856846076,
      "kl": 0.0,
      "learning_rate": 4.67294610151753e-07,
      "logps/chosen": -211.11561584472656,
      "logps/rejected": -253.8684539794922,
      "loss": 0.1853,
      "rewards/chosen": -0.8592585325241089,
      "rewards/margins": 1.5102192163467407,
      "rewards/rejected": -2.3694777488708496,
      "step": 125
    },
    {
      "epoch": 0.07,
      "grad_norm": 11.616081872243173,
      "kl": 0.0,
      "learning_rate": 4.6703296703296704e-07,
      "logps/chosen": -198.94691467285156,
      "logps/rejected": -280.16485595703125,
      "loss": 0.1689,
      "rewards/chosen": -0.6960778832435608,
      "rewards/margins": 2.7315070629119873,
      "rewards/rejected": -3.4275848865509033,
      "step": 126
    },
    {
      "epoch": 0.07,
      "grad_norm": 9.393491633545926,
      "kl": 0.0,
      "learning_rate": 4.6677132391418103e-07,
      "logps/chosen": -237.6060333251953,
      "logps/rejected": -240.0240020751953,
      "loss": 0.2003,
      "rewards/chosen": -0.7763846516609192,
      "rewards/margins": 1.804600715637207,
      "rewards/rejected": -2.5809853076934814,
      "step": 127
    },
    {
      "epoch": 0.07,
      "grad_norm": 10.566838825942087,
      "kl": 0.0,
      "learning_rate": 4.6650968079539503e-07,
      "logps/chosen": -266.26409912109375,
      "logps/rejected": -245.97015380859375,
      "loss": 0.179,
      "rewards/chosen": -0.013315518386662006,
      "rewards/margins": 2.7922699451446533,
      "rewards/rejected": -2.8055853843688965,
      "step": 128
    },
    {
      "epoch": 0.07,
      "grad_norm": 14.386798112694665,
      "kl": 0.0,
      "learning_rate": 4.662480376766091e-07,
      "logps/chosen": -259.3221740722656,
      "logps/rejected": -280.12982177734375,
      "loss": 0.2262,
      "rewards/chosen": 0.06482791900634766,
      "rewards/margins": 1.3292893171310425,
      "rewards/rejected": -1.2644613981246948,
      "step": 129
    },
    {
      "epoch": 0.07,
      "grad_norm": 10.761953162868792,
      "kl": 0.0,
      "learning_rate": 4.6598639455782313e-07,
      "logps/chosen": -193.20936584472656,
      "logps/rejected": -270.4673767089844,
      "loss": 0.1212,
      "rewards/chosen": -0.4515216648578644,
      "rewards/margins": 2.8607840538024902,
      "rewards/rejected": -3.3123056888580322,
      "step": 130
    },
    {
      "epoch": 0.07,
      "grad_norm": 12.751720769362379,
      "kl": 0.0,
      "learning_rate": 4.6572475143903713e-07,
      "logps/chosen": -235.50526428222656,
      "logps/rejected": -217.54153442382812,
      "loss": 0.2364,
      "rewards/chosen": 0.6571396589279175,
      "rewards/margins": 2.786506175994873,
      "rewards/rejected": -2.129366636276245,
      "step": 131
    },
    {
      "epoch": 0.07,
      "grad_norm": 10.524940036364148,
      "kl": 0.0,
      "learning_rate": 4.654631083202512e-07,
      "logps/chosen": -200.957275390625,
      "logps/rejected": -260.49444580078125,
      "loss": 0.1259,
      "rewards/chosen": -0.29692986607551575,
      "rewards/margins": 2.9654293060302734,
      "rewards/rejected": -3.262359142303467,
      "step": 132
    },
    {
      "epoch": 0.07,
      "grad_norm": 10.735468201198927,
      "kl": 0.0,
      "learning_rate": 4.652014652014652e-07,
      "logps/chosen": -243.689208984375,
      "logps/rejected": -259.3601379394531,
      "loss": 0.1994,
      "rewards/chosen": -0.13563716411590576,
      "rewards/margins": 2.890209674835205,
      "rewards/rejected": -3.0258467197418213,
      "step": 133
    },
    {
      "epoch": 0.07,
      "grad_norm": 13.921041963760125,
      "kl": 0.0,
      "learning_rate": 4.6493982208267917e-07,
      "logps/chosen": -260.98651123046875,
      "logps/rejected": -302.0372314453125,
      "loss": 0.2204,
      "rewards/chosen": -0.5615333318710327,
      "rewards/margins": 1.4565614461898804,
      "rewards/rejected": -2.018094778060913,
      "step": 134
    },
    {
      "epoch": 0.07,
      "grad_norm": 8.886703505939792,
      "kl": 0.0,
      "learning_rate": 4.646781789638932e-07,
      "logps/chosen": -219.5489501953125,
      "logps/rejected": -222.55165100097656,
      "loss": 0.2585,
      "rewards/chosen": -2.1579434871673584,
      "rewards/margins": -0.29236698150634766,
      "rewards/rejected": -1.8655765056610107,
      "step": 135
    },
    {
      "epoch": 0.07,
      "grad_norm": 12.360378101489804,
      "kl": 0.0,
      "learning_rate": 4.644165358451073e-07,
      "logps/chosen": -200.14247131347656,
      "logps/rejected": -243.30284118652344,
      "loss": 0.2047,
      "rewards/chosen": -0.22793890535831451,
      "rewards/margins": 2.7516727447509766,
      "rewards/rejected": -2.97961163520813,
      "step": 136
    },
    {
      "epoch": 0.07,
      "grad_norm": 11.520891149644566,
      "kl": 0.0,
      "learning_rate": 4.641548927263213e-07,
      "logps/chosen": -212.42518615722656,
      "logps/rejected": -271.6721496582031,
      "loss": 0.1469,
      "rewards/chosen": -0.4246146082878113,
      "rewards/margins": 3.187201499938965,
      "rewards/rejected": -3.611816167831421,
      "step": 137
    },
    {
      "epoch": 0.07,
      "grad_norm": 8.17959898500212,
      "kl": 0.0,
      "learning_rate": 4.6389324960753527e-07,
      "logps/chosen": -195.15431213378906,
      "logps/rejected": -255.0457000732422,
      "loss": 0.2136,
      "rewards/chosen": -0.6788855791091919,
      "rewards/margins": 2.8008108139038086,
      "rewards/rejected": -3.47969651222229,
      "step": 138
    },
    {
      "epoch": 0.07,
      "grad_norm": 10.843997894954787,
      "kl": 0.0,
      "learning_rate": 4.636316064887493e-07,
      "logps/chosen": -201.34103393554688,
      "logps/rejected": -312.7411804199219,
      "loss": 0.186,
      "rewards/chosen": -1.7989752292633057,
      "rewards/margins": 2.105161428451538,
      "rewards/rejected": -3.9041366577148438,
      "step": 139
    },
    {
      "epoch": 0.07,
      "grad_norm": 12.401485298711776,
      "kl": 0.0,
      "learning_rate": 4.6336996336996337e-07,
      "logps/chosen": -174.31419372558594,
      "logps/rejected": -279.531494140625,
      "loss": 0.1899,
      "rewards/chosen": -0.8827177286148071,
      "rewards/margins": 1.6950773000717163,
      "rewards/rejected": -2.5777950286865234,
      "step": 140
    },
    {
      "epoch": 0.07,
      "grad_norm": 12.876518397686489,
      "kl": 0.0,
      "learning_rate": 4.6310832025117737e-07,
      "logps/chosen": -219.05679321289062,
      "logps/rejected": -270.2620544433594,
      "loss": 0.2003,
      "rewards/chosen": -0.21344774961471558,
      "rewards/margins": 2.0357961654663086,
      "rewards/rejected": -2.249243974685669,
      "step": 141
    },
    {
      "epoch": 0.07,
      "grad_norm": 10.774258331144129,
      "kl": 0.0,
      "learning_rate": 4.628466771323914e-07,
      "logps/chosen": -212.2626495361328,
      "logps/rejected": -283.8631286621094,
      "loss": 0.1711,
      "rewards/chosen": -0.1372440755367279,
      "rewards/margins": 2.4757416248321533,
      "rewards/rejected": -2.612985610961914,
      "step": 142
    },
    {
      "epoch": 0.07,
      "grad_norm": 8.790512175321405,
      "kl": 0.0,
      "learning_rate": 4.625850340136054e-07,
      "logps/chosen": -232.13316345214844,
      "logps/rejected": -223.1201934814453,
      "loss": 0.1773,
      "rewards/chosen": -0.856933057308197,
      "rewards/margins": 2.2891058921813965,
      "rewards/rejected": -3.1460390090942383,
      "step": 143
    },
    {
      "epoch": 0.08,
      "grad_norm": 10.352466882768905,
      "kl": 0.0,
      "learning_rate": 4.623233908948194e-07,
      "logps/chosen": -184.80145263671875,
      "logps/rejected": -193.4806671142578,
      "loss": 0.215,
      "rewards/chosen": -1.3604167699813843,
      "rewards/margins": 0.628121018409729,
      "rewards/rejected": -1.9885377883911133,
      "step": 144
    },
    {
      "epoch": 0.08,
      "grad_norm": 7.163092288844522,
      "kl": 0.0,
      "learning_rate": 4.6206174777603346e-07,
      "logps/chosen": -202.53546142578125,
      "logps/rejected": -254.51487731933594,
      "loss": 0.0969,
      "rewards/chosen": -0.07566285133361816,
      "rewards/margins": 4.109183311462402,
      "rewards/rejected": -4.1848464012146,
      "step": 145
    },
    {
      "epoch": 0.08,
      "grad_norm": 12.265660907944843,
      "kl": 0.0,
      "learning_rate": 4.618001046572475e-07,
      "logps/chosen": -265.9979553222656,
      "logps/rejected": -339.3370666503906,
      "loss": 0.1413,
      "rewards/chosen": 0.027865849435329437,
      "rewards/margins": 4.2701802253723145,
      "rewards/rejected": -4.242314338684082,
      "step": 146
    },
    {
      "epoch": 0.08,
      "grad_norm": 13.038939425208627,
      "kl": 0.0,
      "learning_rate": 4.6153846153846156e-07,
      "logps/chosen": -262.6026306152344,
      "logps/rejected": -261.2329406738281,
      "loss": 0.1645,
      "rewards/chosen": -0.25406259298324585,
      "rewards/margins": 3.3194682598114014,
      "rewards/rejected": -3.573530912399292,
      "step": 147
    },
    {
      "epoch": 0.08,
      "grad_norm": 13.392666406051779,
      "kl": 0.0,
      "learning_rate": 4.612768184196755e-07,
      "logps/chosen": -265.6265563964844,
      "logps/rejected": -281.2951354980469,
      "loss": 0.2543,
      "rewards/chosen": 0.6286525130271912,
      "rewards/margins": 3.1757638454437256,
      "rewards/rejected": -2.5471112728118896,
      "step": 148
    },
    {
      "epoch": 0.08,
      "grad_norm": 14.164191990445923,
      "kl": 0.0,
      "learning_rate": 4.6101517530088956e-07,
      "logps/chosen": -280.87567138671875,
      "logps/rejected": -298.3075866699219,
      "loss": 0.2937,
      "rewards/chosen": -1.144802212715149,
      "rewards/margins": 0.6895921230316162,
      "rewards/rejected": -1.8343943357467651,
      "step": 149
    },
    {
      "epoch": 0.08,
      "grad_norm": 10.347666390603884,
      "kl": 0.0,
      "learning_rate": 4.607535321821036e-07,
      "logps/chosen": -227.58335876464844,
      "logps/rejected": -239.180908203125,
      "loss": 0.1543,
      "rewards/chosen": -1.3316487073898315,
      "rewards/margins": 0.7044416666030884,
      "rewards/rejected": -2.03609037399292,
      "step": 150
    },
    {
      "epoch": 0.08,
      "grad_norm": 11.52825621590645,
      "kl": 0.0,
      "learning_rate": 4.604918890633176e-07,
      "logps/chosen": -278.8197326660156,
      "logps/rejected": -309.47271728515625,
      "loss": 0.2073,
      "rewards/chosen": -0.36525189876556396,
      "rewards/margins": 3.581759452819824,
      "rewards/rejected": -3.9470112323760986,
      "step": 151
    },
    {
      "epoch": 0.08,
      "grad_norm": 10.00307147974759,
      "kl": 0.0,
      "learning_rate": 4.6023024594453166e-07,
      "logps/chosen": -232.21923828125,
      "logps/rejected": -288.6890563964844,
      "loss": 0.1359,
      "rewards/chosen": -1.7685518264770508,
      "rewards/margins": 1.4930996894836426,
      "rewards/rejected": -3.2616515159606934,
      "step": 152
    },
    {
      "epoch": 0.08,
      "grad_norm": 9.048295062554471,
      "kl": 0.0,
      "learning_rate": 4.5996860282574565e-07,
      "logps/chosen": -263.74456787109375,
      "logps/rejected": -182.83853149414062,
      "loss": 0.1225,
      "rewards/chosen": 0.26467177271842957,
      "rewards/margins": 3.1499152183532715,
      "rewards/rejected": -2.8852434158325195,
      "step": 153
    },
    {
      "epoch": 0.08,
      "grad_norm": 9.505705109305136,
      "kl": 0.0,
      "learning_rate": 4.5970695970695965e-07,
      "logps/chosen": -239.68685913085938,
      "logps/rejected": -284.68927001953125,
      "loss": 0.228,
      "rewards/chosen": -1.0681113004684448,
      "rewards/margins": 0.38854992389678955,
      "rewards/rejected": -1.4566612243652344,
      "step": 154
    },
    {
      "epoch": 0.08,
      "grad_norm": 12.907956927820472,
      "kl": 0.0,
      "learning_rate": 4.594453165881737e-07,
      "logps/chosen": -171.40005493164062,
      "logps/rejected": -231.8187255859375,
      "loss": 0.2271,
      "rewards/chosen": -0.5379889607429504,
      "rewards/margins": 2.1988117694854736,
      "rewards/rejected": -2.7368006706237793,
      "step": 155
    },
    {
      "epoch": 0.08,
      "grad_norm": 10.134785294701812,
      "kl": 0.0,
      "learning_rate": 4.5918367346938775e-07,
      "logps/chosen": -206.3992462158203,
      "logps/rejected": -202.78799438476562,
      "loss": 0.1833,
      "rewards/chosen": -0.1859370917081833,
      "rewards/margins": 2.8349618911743164,
      "rewards/rejected": -3.0208990573883057,
      "step": 156
    },
    {
      "epoch": 0.08,
      "grad_norm": 10.654899640372967,
      "kl": 0.0,
      "learning_rate": 4.589220303506018e-07,
      "logps/chosen": -226.39356994628906,
      "logps/rejected": -253.27488708496094,
      "loss": 0.2065,
      "rewards/chosen": -0.936338484287262,
      "rewards/margins": 1.9939072132110596,
      "rewards/rejected": -2.9302456378936768,
      "step": 157
    },
    {
      "epoch": 0.08,
      "grad_norm": 10.940456012048248,
      "kl": 0.0,
      "learning_rate": 4.5866038723181575e-07,
      "logps/chosen": -224.9565887451172,
      "logps/rejected": -265.723388671875,
      "loss": 0.2093,
      "rewards/chosen": -1.0069631338119507,
      "rewards/margins": 1.279760718345642,
      "rewards/rejected": -2.2867238521575928,
      "step": 158
    },
    {
      "epoch": 0.08,
      "grad_norm": 13.331073300797962,
      "kl": 0.0,
      "learning_rate": 4.583987441130298e-07,
      "logps/chosen": -199.77915954589844,
      "logps/rejected": -303.58538818359375,
      "loss": 0.1757,
      "rewards/chosen": -0.519672155380249,
      "rewards/margins": 1.5667927265167236,
      "rewards/rejected": -2.0864648818969727,
      "step": 159
    },
    {
      "epoch": 0.08,
      "grad_norm": 12.759132153782163,
      "kl": 0.0,
      "learning_rate": 4.5813710099424385e-07,
      "logps/chosen": -263.2257385253906,
      "logps/rejected": -216.82395935058594,
      "loss": 0.2497,
      "rewards/chosen": -0.19666321575641632,
      "rewards/margins": 1.9927500486373901,
      "rewards/rejected": -2.18941330909729,
      "step": 160
    },
    {
      "epoch": 0.08,
      "grad_norm": 13.295115337094142,
      "kl": 0.0,
      "learning_rate": 4.5787545787545784e-07,
      "logps/chosen": -228.7009735107422,
      "logps/rejected": -295.6015319824219,
      "loss": 0.2364,
      "rewards/chosen": -0.8198361992835999,
      "rewards/margins": 0.8233547806739807,
      "rewards/rejected": -1.6431909799575806,
      "step": 161
    },
    {
      "epoch": 0.08,
      "grad_norm": 14.82680643745037,
      "kl": 0.0,
      "learning_rate": 4.576138147566719e-07,
      "logps/chosen": -260.577880859375,
      "logps/rejected": -253.9260711669922,
      "loss": 0.2625,
      "rewards/chosen": -0.1303100883960724,
      "rewards/margins": 2.1792666912078857,
      "rewards/rejected": -2.3095767498016357,
      "step": 162
    },
    {
      "epoch": 0.09,
      "grad_norm": 13.208958389079216,
      "kl": 0.0,
      "learning_rate": 4.5735217163788594e-07,
      "logps/chosen": -190.9998016357422,
      "logps/rejected": -276.3247985839844,
      "loss": 0.1975,
      "rewards/chosen": 0.03442925959825516,
      "rewards/margins": 2.56953763961792,
      "rewards/rejected": -2.5351083278656006,
      "step": 163
    },
    {
      "epoch": 0.09,
      "grad_norm": 11.80477720551951,
      "kl": 0.0,
      "learning_rate": 4.570905285190999e-07,
      "logps/chosen": -246.15234375,
      "logps/rejected": -248.68072509765625,
      "loss": 0.2212,
      "rewards/chosen": -0.5984467267990112,
      "rewards/margins": 1.7412387132644653,
      "rewards/rejected": -2.3396854400634766,
      "step": 164
    },
    {
      "epoch": 0.09,
      "grad_norm": 9.00263121063917,
      "kl": 0.0,
      "learning_rate": 4.5682888540031394e-07,
      "logps/chosen": -252.92127990722656,
      "logps/rejected": -215.0977325439453,
      "loss": 0.1732,
      "rewards/chosen": -0.49442756175994873,
      "rewards/margins": 2.391329288482666,
      "rewards/rejected": -2.885756731033325,
      "step": 165
    },
    {
      "epoch": 0.09,
      "grad_norm": 9.7104117892985,
      "kl": 0.0,
      "learning_rate": 4.56567242281528e-07,
      "logps/chosen": -286.4942626953125,
      "logps/rejected": -277.3382873535156,
      "loss": 0.1979,
      "rewards/chosen": -0.5238850712776184,
      "rewards/margins": 2.418689489364624,
      "rewards/rejected": -2.9425745010375977,
      "step": 166
    },
    {
      "epoch": 0.09,
      "grad_norm": 9.591639733233848,
      "kl": 0.0,
      "learning_rate": 4.56305599162742e-07,
      "logps/chosen": -237.4594268798828,
      "logps/rejected": -217.5932159423828,
      "loss": 0.1465,
      "rewards/chosen": -0.8079670071601868,
      "rewards/margins": 1.809727668762207,
      "rewards/rejected": -2.617694616317749,
      "step": 167
    },
    {
      "epoch": 0.09,
      "grad_norm": 11.459074832211314,
      "kl": 0.0,
      "learning_rate": 4.5604395604395604e-07,
      "logps/chosen": -220.34298706054688,
      "logps/rejected": -245.24441528320312,
      "loss": 0.17,
      "rewards/chosen": 0.019128350540995598,
      "rewards/margins": 2.7078402042388916,
      "rewards/rejected": -2.6887118816375732,
      "step": 168
    },
    {
      "epoch": 0.09,
      "grad_norm": 15.063963939999352,
      "kl": 0.0,
      "learning_rate": 4.5578231292517003e-07,
      "logps/chosen": -297.51300048828125,
      "logps/rejected": -251.10533142089844,
      "loss": 0.1753,
      "rewards/chosen": 0.8247194886207581,
      "rewards/margins": 2.683767557144165,
      "rewards/rejected": -1.8590481281280518,
      "step": 169
    },
    {
      "epoch": 0.09,
      "grad_norm": 9.142681234947581,
      "kl": 0.0,
      "learning_rate": 4.555206698063841e-07,
      "logps/chosen": -300.2449951171875,
      "logps/rejected": -224.29713439941406,
      "loss": 0.2087,
      "rewards/chosen": -0.8797904253005981,
      "rewards/margins": 1.0547667741775513,
      "rewards/rejected": -1.9345571994781494,
      "step": 170
    },
    {
      "epoch": 0.09,
      "grad_norm": 10.227907117502706,
      "kl": 0.0,
      "learning_rate": 4.552590266875981e-07,
      "logps/chosen": -224.4203338623047,
      "logps/rejected": -315.0204162597656,
      "loss": 0.1784,
      "rewards/chosen": -0.9637618064880371,
      "rewards/margins": 2.0699081420898438,
      "rewards/rejected": -3.033669948577881,
      "step": 171
    },
    {
      "epoch": 0.09,
      "grad_norm": 11.184644215978503,
      "kl": 0.0,
      "learning_rate": 4.5499738356881213e-07,
      "logps/chosen": -232.22695922851562,
      "logps/rejected": -202.02688598632812,
      "loss": 0.2471,
      "rewards/chosen": -1.0796171426773071,
      "rewards/margins": 1.041359305381775,
      "rewards/rejected": -2.120976448059082,
      "step": 172
    },
    {
      "epoch": 0.09,
      "grad_norm": 12.858302396434425,
      "kl": 0.0,
      "learning_rate": 4.547357404500262e-07,
      "logps/chosen": -231.92547607421875,
      "logps/rejected": -284.2903137207031,
      "loss": 0.1662,
      "rewards/chosen": 0.22813068330287933,
      "rewards/margins": 2.726292610168457,
      "rewards/rejected": -2.498162031173706,
      "step": 173
    },
    {
      "epoch": 0.09,
      "grad_norm": 10.830471355823923,
      "kl": 0.0,
      "learning_rate": 4.5447409733124013e-07,
      "logps/chosen": -245.82032775878906,
      "logps/rejected": -204.56790161132812,
      "loss": 0.2141,
      "rewards/chosen": 0.45919883251190186,
      "rewards/margins": 2.118945598602295,
      "rewards/rejected": -1.6597468852996826,
      "step": 174
    },
    {
      "epoch": 0.09,
      "grad_norm": 11.540820927447694,
      "kl": 0.0,
      "learning_rate": 4.542124542124542e-07,
      "logps/chosen": -221.0320587158203,
      "logps/rejected": -244.3851318359375,
      "loss": 0.1404,
      "rewards/chosen": 0.015136350877583027,
      "rewards/margins": 2.4611456394195557,
      "rewards/rejected": -2.446009397506714,
      "step": 175
    },
    {
      "epoch": 0.09,
      "grad_norm": 9.370935336042336,
      "kl": 0.0,
      "learning_rate": 4.5395081109366823e-07,
      "logps/chosen": -148.11834716796875,
      "logps/rejected": -254.36962890625,
      "loss": 0.1237,
      "rewards/chosen": 0.47263994812965393,
      "rewards/margins": 3.6257412433624268,
      "rewards/rejected": -3.1531012058258057,
      "step": 176
    },
    {
      "epoch": 0.09,
      "grad_norm": 9.524027879440371,
      "kl": 0.0,
      "learning_rate": 4.536891679748822e-07,
      "logps/chosen": -218.85006713867188,
      "logps/rejected": -268.6586608886719,
      "loss": 0.1249,
      "rewards/chosen": -1.152251958847046,
      "rewards/margins": 1.3508758544921875,
      "rewards/rejected": -2.5031278133392334,
      "step": 177
    },
    {
      "epoch": 0.09,
      "grad_norm": 10.097839005642474,
      "kl": 0.0,
      "learning_rate": 4.534275248560963e-07,
      "logps/chosen": -140.55996704101562,
      "logps/rejected": -265.65789794921875,
      "loss": 0.279,
      "rewards/chosen": -0.11726745963096619,
      "rewards/margins": 1.562013030052185,
      "rewards/rejected": -1.6792805194854736,
      "step": 178
    },
    {
      "epoch": 0.09,
      "grad_norm": 11.353665853013293,
      "kl": 0.0,
      "learning_rate": 4.5316588173731027e-07,
      "logps/chosen": -168.6680145263672,
      "logps/rejected": -309.8199462890625,
      "loss": 0.1995,
      "rewards/chosen": -0.32298406958580017,
      "rewards/margins": 1.8489097356796265,
      "rewards/rejected": -2.171893835067749,
      "step": 179
    },
    {
      "epoch": 0.09,
      "grad_norm": 10.693119887330608,
      "kl": 0.0,
      "learning_rate": 4.529042386185243e-07,
      "logps/chosen": -262.695556640625,
      "logps/rejected": -234.45143127441406,
      "loss": 0.2569,
      "rewards/chosen": -0.22278353571891785,
      "rewards/margins": 2.558516502380371,
      "rewards/rejected": -2.7813000679016113,
      "step": 180
    },
    {
      "epoch": 0.09,
      "grad_norm": 13.107582728245008,
      "kl": 0.0,
      "learning_rate": 4.526425954997383e-07,
      "logps/chosen": -244.29647827148438,
      "logps/rejected": -291.70941162109375,
      "loss": 0.2651,
      "rewards/chosen": -0.5042144656181335,
      "rewards/margins": 2.197082757949829,
      "rewards/rejected": -2.7012972831726074,
      "step": 181
    },
    {
      "epoch": 0.1,
      "grad_norm": 9.773171918884088,
      "kl": 0.0,
      "learning_rate": 4.5238095238095237e-07,
      "logps/chosen": -197.32012939453125,
      "logps/rejected": -293.9897766113281,
      "loss": 0.1873,
      "rewards/chosen": -0.5700012445449829,
      "rewards/margins": 4.046950340270996,
      "rewards/rejected": -4.6169514656066895,
      "step": 182
    },
    {
      "epoch": 0.1,
      "grad_norm": 11.20040922510821,
      "kl": 0.0,
      "learning_rate": 4.521193092621664e-07,
      "logps/chosen": -194.4012908935547,
      "logps/rejected": -223.0313720703125,
      "loss": 0.1483,
      "rewards/chosen": 0.47125691175460815,
      "rewards/margins": 2.2409911155700684,
      "rewards/rejected": -1.7697341442108154,
      "step": 183
    },
    {
      "epoch": 0.1,
      "grad_norm": 12.989173617269795,
      "kl": 0.0,
      "learning_rate": 4.5185766614338037e-07,
      "logps/chosen": -171.93673706054688,
      "logps/rejected": -316.18316650390625,
      "loss": 0.216,
      "rewards/chosen": -0.1644737273454666,
      "rewards/margins": 0.4819384217262268,
      "rewards/rejected": -0.6464121341705322,
      "step": 184
    },
    {
      "epoch": 0.1,
      "grad_norm": 13.329019681893538,
      "kl": 0.0,
      "learning_rate": 4.515960230245944e-07,
      "logps/chosen": -215.2859649658203,
      "logps/rejected": -258.4815368652344,
      "loss": 0.1873,
      "rewards/chosen": 0.19089780747890472,
      "rewards/margins": 3.2682807445526123,
      "rewards/rejected": -3.077383041381836,
      "step": 185
    },
    {
      "epoch": 0.1,
      "grad_norm": 9.903441953636646,
      "kl": 0.0,
      "learning_rate": 4.5133437990580847e-07,
      "logps/chosen": -203.1886444091797,
      "logps/rejected": -219.4387969970703,
      "loss": 0.1725,
      "rewards/chosen": 0.7246238589286804,
      "rewards/margins": 1.6935207843780518,
      "rewards/rejected": -0.9688969254493713,
      "step": 186
    },
    {
      "epoch": 0.1,
      "grad_norm": 10.026171474980584,
      "kl": 0.0,
      "learning_rate": 4.5107273678702246e-07,
      "logps/chosen": -237.39012145996094,
      "logps/rejected": -313.46038818359375,
      "loss": 0.1848,
      "rewards/chosen": 0.13696634769439697,
      "rewards/margins": 2.4037017822265625,
      "rewards/rejected": -2.266735315322876,
      "step": 187
    },
    {
      "epoch": 0.1,
      "grad_norm": 13.743945884823951,
      "kl": 0.0,
      "learning_rate": 4.508110936682365e-07,
      "logps/chosen": -226.96961975097656,
      "logps/rejected": -239.95713806152344,
      "loss": 0.0994,
      "rewards/chosen": 0.7345108389854431,
      "rewards/margins": 2.190957546234131,
      "rewards/rejected": -1.4564467668533325,
      "step": 188
    },
    {
      "epoch": 0.1,
      "grad_norm": 11.111688156500954,
      "kl": 0.0,
      "learning_rate": 4.5054945054945056e-07,
      "logps/chosen": -228.42758178710938,
      "logps/rejected": -203.33209228515625,
      "loss": 0.1395,
      "rewards/chosen": 0.4875214099884033,
      "rewards/margins": 1.7979084253311157,
      "rewards/rejected": -1.3103870153427124,
      "step": 189
    },
    {
      "epoch": 0.1,
      "grad_norm": 9.470046480370337,
      "kl": 0.0,
      "learning_rate": 4.5028780743066456e-07,
      "logps/chosen": -207.3546905517578,
      "logps/rejected": -231.11886596679688,
      "loss": 0.1262,
      "rewards/chosen": -0.461222380399704,
      "rewards/margins": 1.9130089282989502,
      "rewards/rejected": -2.3742313385009766,
      "step": 190
    },
    {
      "epoch": 0.1,
      "grad_norm": 12.783425591778233,
      "kl": 0.0,
      "learning_rate": 4.5002616431187856e-07,
      "logps/chosen": -212.4601593017578,
      "logps/rejected": -252.49920654296875,
      "loss": 0.1552,
      "rewards/chosen": 0.14723794162273407,
      "rewards/margins": 1.8059498071670532,
      "rewards/rejected": -1.6587119102478027,
      "step": 191
    },
    {
      "epoch": 0.1,
      "grad_norm": 11.876829640439261,
      "kl": 0.0,
      "learning_rate": 4.497645211930926e-07,
      "logps/chosen": -331.9737548828125,
      "logps/rejected": -216.70932006835938,
      "loss": 0.1592,
      "rewards/chosen": 0.7978419065475464,
      "rewards/margins": 3.4641337394714355,
      "rewards/rejected": -2.6662919521331787,
      "step": 192
    },
    {
      "epoch": 0.1,
      "grad_norm": 11.18786423314498,
      "kl": 0.0,
      "learning_rate": 4.4950287807430666e-07,
      "logps/chosen": -210.8509521484375,
      "logps/rejected": -276.6178283691406,
      "loss": 0.1855,
      "rewards/chosen": 0.7132515907287598,
      "rewards/margins": 3.511439561843872,
      "rewards/rejected": -2.7981879711151123,
      "step": 193
    },
    {
      "epoch": 0.1,
      "grad_norm": 11.51339539474961,
      "kl": 0.0,
      "learning_rate": 4.4924123495552066e-07,
      "logps/chosen": -193.66836547851562,
      "logps/rejected": -206.64353942871094,
      "loss": 0.2209,
      "rewards/chosen": -0.7599809765815735,
      "rewards/margins": 0.2910911440849304,
      "rewards/rejected": -1.051072120666504,
      "step": 194
    },
    {
      "epoch": 0.1,
      "grad_norm": 9.775297052636276,
      "kl": 0.0,
      "learning_rate": 4.4897959183673465e-07,
      "logps/chosen": -210.68817138671875,
      "logps/rejected": -269.3539123535156,
      "loss": 0.1624,
      "rewards/chosen": 0.5124328136444092,
      "rewards/margins": 3.528744697570801,
      "rewards/rejected": -3.0163118839263916,
      "step": 195
    },
    {
      "epoch": 0.1,
      "grad_norm": 13.388916890187613,
      "kl": 0.0,
      "learning_rate": 4.487179487179487e-07,
      "logps/chosen": -286.09918212890625,
      "logps/rejected": -238.72901916503906,
      "loss": 0.2343,
      "rewards/chosen": -0.26804402470588684,
      "rewards/margins": 1.2514140605926514,
      "rewards/rejected": -1.5194580554962158,
      "step": 196
    },
    {
      "epoch": 0.1,
      "grad_norm": 11.098868865803937,
      "kl": 0.0,
      "learning_rate": 4.484563055991627e-07,
      "logps/chosen": -247.14202880859375,
      "logps/rejected": -196.82537841796875,
      "loss": 0.157,
      "rewards/chosen": -0.33019348978996277,
      "rewards/margins": 2.0103812217712402,
      "rewards/rejected": -2.3405747413635254,
      "step": 197
    },
    {
      "epoch": 0.1,
      "grad_norm": 14.223147581105783,
      "kl": 0.0,
      "learning_rate": 4.4819466248037675e-07,
      "logps/chosen": -204.22280883789062,
      "logps/rejected": -286.9610900878906,
      "loss": 0.237,
      "rewards/chosen": -0.013581500388681889,
      "rewards/margins": 1.4753634929656982,
      "rewards/rejected": -1.4889450073242188,
      "step": 198
    },
    {
      "epoch": 0.1,
      "grad_norm": 14.41947331635678,
      "kl": 0.0,
      "learning_rate": 4.479330193615908e-07,
      "logps/chosen": -213.51507568359375,
      "logps/rejected": -270.748046875,
      "loss": 0.1949,
      "rewards/chosen": -0.7232166528701782,
      "rewards/margins": 2.0983409881591797,
      "rewards/rejected": -2.8215577602386475,
      "step": 199
    },
    {
      "epoch": 0.1,
      "grad_norm": 9.05336870531292,
      "kl": 0.0,
      "learning_rate": 4.476713762428048e-07,
      "logps/chosen": -266.99261474609375,
      "logps/rejected": -215.208740234375,
      "loss": 0.164,
      "rewards/chosen": 0.1612011045217514,
      "rewards/margins": 1.967577338218689,
      "rewards/rejected": -1.8063762187957764,
      "step": 200
    },
    {
      "epoch": 0.11,
      "grad_norm": 12.145917811167815,
      "kl": 0.0,
      "learning_rate": 4.474097331240188e-07,
      "logps/chosen": -311.310546875,
      "logps/rejected": -207.57504272460938,
      "loss": 0.198,
      "rewards/chosen": 0.33403000235557556,
      "rewards/margins": 2.6162540912628174,
      "rewards/rejected": -2.282224178314209,
      "step": 201
    },
    {
      "epoch": 0.11,
      "grad_norm": 10.620881317672168,
      "kl": 0.0,
      "learning_rate": 4.4714809000523285e-07,
      "logps/chosen": -250.87953186035156,
      "logps/rejected": -275.86480712890625,
      "loss": 0.1359,
      "rewards/chosen": 0.3162269592285156,
      "rewards/margins": 1.990512728691101,
      "rewards/rejected": -1.6742857694625854,
      "step": 202
    },
    {
      "epoch": 0.11,
      "grad_norm": 10.496041019849555,
      "kl": 0.0,
      "learning_rate": 4.468864468864469e-07,
      "logps/chosen": -247.9547119140625,
      "logps/rejected": -217.61874389648438,
      "loss": 0.203,
      "rewards/chosen": -1.653024673461914,
      "rewards/margins": 0.7888014316558838,
      "rewards/rejected": -2.441826105117798,
      "step": 203
    },
    {
      "epoch": 0.11,
      "grad_norm": 11.99618040191761,
      "kl": 0.0,
      "learning_rate": 4.466248037676609e-07,
      "logps/chosen": -150.46995544433594,
      "logps/rejected": -319.3192138671875,
      "loss": 0.2054,
      "rewards/chosen": -0.28798094391822815,
      "rewards/margins": 1.984279751777649,
      "rewards/rejected": -2.2722606658935547,
      "step": 204
    },
    {
      "epoch": 0.11,
      "grad_norm": 11.06635292149166,
      "kl": 0.0,
      "learning_rate": 4.463631606488749e-07,
      "logps/chosen": -206.5174560546875,
      "logps/rejected": -254.26016235351562,
      "loss": 0.1693,
      "rewards/chosen": -0.9574307799339294,
      "rewards/margins": 2.373122453689575,
      "rewards/rejected": -3.3305532932281494,
      "step": 205
    },
    {
      "epoch": 0.11,
      "grad_norm": 9.1313149728796,
      "kl": 0.0,
      "learning_rate": 4.4610151753008894e-07,
      "logps/chosen": -211.5845489501953,
      "logps/rejected": -266.97222900390625,
      "loss": 0.2363,
      "rewards/chosen": -0.3957107961177826,
      "rewards/margins": 3.1181278228759766,
      "rewards/rejected": -3.513838529586792,
      "step": 206
    },
    {
      "epoch": 0.11,
      "grad_norm": 13.851054073575309,
      "kl": 0.0,
      "learning_rate": 4.4583987441130294e-07,
      "logps/chosen": -266.7466125488281,
      "logps/rejected": -223.49757385253906,
      "loss": 0.2842,
      "rewards/chosen": -1.3039528131484985,
      "rewards/margins": 0.7924333810806274,
      "rewards/rejected": -2.096386194229126,
      "step": 207
    },
    {
      "epoch": 0.11,
      "grad_norm": 11.650743591966863,
      "kl": 0.0,
      "learning_rate": 4.45578231292517e-07,
      "logps/chosen": -183.24017333984375,
      "logps/rejected": -276.4227294921875,
      "loss": 0.1615,
      "rewards/chosen": -0.3646261990070343,
      "rewards/margins": 2.279031276702881,
      "rewards/rejected": -2.6436574459075928,
      "step": 208
    },
    {
      "epoch": 0.11,
      "grad_norm": 13.362859226103993,
      "kl": 0.0,
      "learning_rate": 4.4531658817373104e-07,
      "logps/chosen": -205.38035583496094,
      "logps/rejected": -269.58526611328125,
      "loss": 0.2259,
      "rewards/chosen": -0.2013118416070938,
      "rewards/margins": 2.4193832874298096,
      "rewards/rejected": -2.620695114135742,
      "step": 209
    },
    {
      "epoch": 0.11,
      "grad_norm": 12.32754631235818,
      "kl": 0.0,
      "learning_rate": 4.45054945054945e-07,
      "logps/chosen": -268.922607421875,
      "logps/rejected": -284.7842102050781,
      "loss": 0.2121,
      "rewards/chosen": -1.0982351303100586,
      "rewards/margins": 1.700211524963379,
      "rewards/rejected": -2.7984466552734375,
      "step": 210
    },
    {
      "epoch": 0.11,
      "grad_norm": 11.666014991292931,
      "kl": 0.0,
      "learning_rate": 4.4479330193615904e-07,
      "logps/chosen": -273.0299987792969,
      "logps/rejected": -326.8552551269531,
      "loss": 0.139,
      "rewards/chosen": -0.8367617726325989,
      "rewards/margins": 2.7738115787506104,
      "rewards/rejected": -3.6105732917785645,
      "step": 211
    },
    {
      "epoch": 0.11,
      "grad_norm": 12.85199720190171,
      "kl": 0.0,
      "learning_rate": 4.445316588173731e-07,
      "logps/chosen": -267.0897216796875,
      "logps/rejected": -248.22039794921875,
      "loss": 0.2089,
      "rewards/chosen": -1.914868950843811,
      "rewards/margins": 1.703445553779602,
      "rewards/rejected": -3.618314504623413,
      "step": 212
    },
    {
      "epoch": 0.11,
      "grad_norm": 13.279155862550748,
      "kl": 0.0,
      "learning_rate": 4.4427001569858714e-07,
      "logps/chosen": -270.5106201171875,
      "logps/rejected": -323.9206848144531,
      "loss": 0.1508,
      "rewards/chosen": 0.43147632479667664,
      "rewards/margins": 4.1532816886901855,
      "rewards/rejected": -3.7218053340911865,
      "step": 213
    },
    {
      "epoch": 0.11,
      "grad_norm": 10.842517342661287,
      "kl": 0.0,
      "learning_rate": 4.4400837257980113e-07,
      "logps/chosen": -251.76222229003906,
      "logps/rejected": -204.5667724609375,
      "loss": 0.2109,
      "rewards/chosen": -0.23020866513252258,
      "rewards/margins": 1.9914796352386475,
      "rewards/rejected": -2.2216882705688477,
      "step": 214
    },
    {
      "epoch": 0.11,
      "grad_norm": 11.17043596208898,
      "kl": 0.0,
      "learning_rate": 4.4374672946101513e-07,
      "logps/chosen": -203.4984588623047,
      "logps/rejected": -343.187255859375,
      "loss": 0.2139,
      "rewards/chosen": -0.3769696354866028,
      "rewards/margins": 2.5140483379364014,
      "rewards/rejected": -2.8910179138183594,
      "step": 215
    },
    {
      "epoch": 0.11,
      "grad_norm": 11.293416573180265,
      "kl": 0.0,
      "learning_rate": 4.434850863422292e-07,
      "logps/chosen": -294.1488037109375,
      "logps/rejected": -282.24981689453125,
      "loss": 0.1393,
      "rewards/chosen": 0.7586994767189026,
      "rewards/margins": 4.094053268432617,
      "rewards/rejected": -3.3353536128997803,
      "step": 216
    },
    {
      "epoch": 0.11,
      "grad_norm": 13.608688968178855,
      "kl": 0.0,
      "learning_rate": 4.432234432234432e-07,
      "logps/chosen": -274.1988220214844,
      "logps/rejected": -293.933837890625,
      "loss": 0.1346,
      "rewards/chosen": -0.13219358026981354,
      "rewards/margins": 2.8667314052581787,
      "rewards/rejected": -2.998924970626831,
      "step": 217
    },
    {
      "epoch": 0.11,
      "grad_norm": 10.83390830521355,
      "kl": 0.0,
      "learning_rate": 4.4296180010465723e-07,
      "logps/chosen": -222.037353515625,
      "logps/rejected": -319.1642150878906,
      "loss": 0.1958,
      "rewards/chosen": -0.27647799253463745,
      "rewards/margins": 2.2584426403045654,
      "rewards/rejected": -2.5349206924438477,
      "step": 218
    },
    {
      "epoch": 0.11,
      "grad_norm": 11.966693070507677,
      "kl": 0.0,
      "learning_rate": 4.427001569858713e-07,
      "logps/chosen": -191.46580505371094,
      "logps/rejected": -366.8327941894531,
      "loss": 0.1951,
      "rewards/chosen": -0.4248928129673004,
      "rewards/margins": 2.9257259368896484,
      "rewards/rejected": -3.350618839263916,
      "step": 219
    },
    {
      "epoch": 0.12,
      "grad_norm": 9.986050551901583,
      "kl": 0.0,
      "learning_rate": 4.424385138670852e-07,
      "logps/chosen": -235.12484741210938,
      "logps/rejected": -248.2144012451172,
      "loss": 0.2251,
      "rewards/chosen": -1.9966895580291748,
      "rewards/margins": 1.9721267223358154,
      "rewards/rejected": -3.9688162803649902,
      "step": 220
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.554269588738498,
      "kl": 0.0,
      "learning_rate": 4.421768707482993e-07,
      "logps/chosen": -239.76820373535156,
      "logps/rejected": -218.79449462890625,
      "loss": 0.2125,
      "rewards/chosen": -1.0601166486740112,
      "rewards/margins": 2.6879591941833496,
      "rewards/rejected": -3.7480757236480713,
      "step": 221
    },
    {
      "epoch": 0.12,
      "grad_norm": 11.807560445400602,
      "kl": 0.0,
      "learning_rate": 4.419152276295133e-07,
      "logps/chosen": -226.56256103515625,
      "logps/rejected": -218.861328125,
      "loss": 0.1385,
      "rewards/chosen": -0.5115461349487305,
      "rewards/margins": 2.6464967727661133,
      "rewards/rejected": -3.1580429077148438,
      "step": 222
    },
    {
      "epoch": 0.12,
      "grad_norm": 11.670267600602166,
      "kl": 0.0,
      "learning_rate": 4.416535845107274e-07,
      "logps/chosen": -282.7430725097656,
      "logps/rejected": -211.0267791748047,
      "loss": 0.2221,
      "rewards/chosen": -1.8116750717163086,
      "rewards/margins": 0.58609938621521,
      "rewards/rejected": -2.3977744579315186,
      "step": 223
    },
    {
      "epoch": 0.12,
      "grad_norm": 11.238377690831841,
      "kl": 0.0,
      "learning_rate": 4.4139194139194137e-07,
      "logps/chosen": -218.9561004638672,
      "logps/rejected": -232.92832946777344,
      "loss": 0.1533,
      "rewards/chosen": -0.23154856264591217,
      "rewards/margins": 1.915379285812378,
      "rewards/rejected": -2.146927833557129,
      "step": 224
    },
    {
      "epoch": 0.12,
      "grad_norm": 13.477552695461938,
      "kl": 0.0,
      "learning_rate": 4.411302982731554e-07,
      "logps/chosen": -241.8193359375,
      "logps/rejected": -243.86459350585938,
      "loss": 0.2513,
      "rewards/chosen": -0.6799749135971069,
      "rewards/margins": 0.9313141107559204,
      "rewards/rejected": -1.6112890243530273,
      "step": 225
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.248025744939582,
      "kl": 0.0,
      "learning_rate": 4.408686551543694e-07,
      "logps/chosen": -255.42225646972656,
      "logps/rejected": -354.8009948730469,
      "loss": 0.1809,
      "rewards/chosen": -0.15073581039905548,
      "rewards/margins": 3.6917855739593506,
      "rewards/rejected": -3.8425214290618896,
      "step": 226
    },
    {
      "epoch": 0.12,
      "grad_norm": 15.177673567635171,
      "kl": 0.0,
      "learning_rate": 4.406070120355834e-07,
      "logps/chosen": -219.69955444335938,
      "logps/rejected": -257.246337890625,
      "loss": 0.2311,
      "rewards/chosen": -0.5061897039413452,
      "rewards/margins": 1.860742449760437,
      "rewards/rejected": -2.3669321537017822,
      "step": 227
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.212253558418395,
      "kl": 0.0,
      "learning_rate": 4.4034536891679747e-07,
      "logps/chosen": -216.00762939453125,
      "logps/rejected": -271.8702392578125,
      "loss": 0.1099,
      "rewards/chosen": -0.29265689849853516,
      "rewards/margins": 2.9313840866088867,
      "rewards/rejected": -3.224040985107422,
      "step": 228
    },
    {
      "epoch": 0.12,
      "grad_norm": 11.734827553729616,
      "kl": 0.0,
      "learning_rate": 4.400837257980115e-07,
      "logps/chosen": -217.1133270263672,
      "logps/rejected": -310.7548522949219,
      "loss": 0.2233,
      "rewards/chosen": -0.2567431330680847,
      "rewards/margins": 2.2228996753692627,
      "rewards/rejected": -2.479642868041992,
      "step": 229
    },
    {
      "epoch": 0.12,
      "grad_norm": 9.41912765209931,
      "kl": 0.0,
      "learning_rate": 4.398220826792255e-07,
      "logps/chosen": -194.0414581298828,
      "logps/rejected": -252.26522827148438,
      "loss": 0.1112,
      "rewards/chosen": 0.5574043989181519,
      "rewards/margins": 4.030994892120361,
      "rewards/rejected": -3.47359037399292,
      "step": 230
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.488015000282592,
      "kl": 0.0,
      "learning_rate": 4.395604395604395e-07,
      "logps/chosen": -188.4119873046875,
      "logps/rejected": -282.2178039550781,
      "loss": 0.1811,
      "rewards/chosen": -0.3539419174194336,
      "rewards/margins": 1.9429447650909424,
      "rewards/rejected": -2.296886682510376,
      "step": 231
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.39486238771726,
      "kl": 0.0,
      "learning_rate": 4.3929879644165356e-07,
      "logps/chosen": -223.63308715820312,
      "logps/rejected": -240.90345764160156,
      "loss": 0.1975,
      "rewards/chosen": -0.5462809205055237,
      "rewards/margins": 1.494751214981079,
      "rewards/rejected": -2.041032075881958,
      "step": 232
    },
    {
      "epoch": 0.12,
      "grad_norm": 11.215577164704541,
      "kl": 0.0,
      "learning_rate": 4.390371533228676e-07,
      "logps/chosen": -208.15684509277344,
      "logps/rejected": -315.0474548339844,
      "loss": 0.2079,
      "rewards/chosen": -1.009997010231018,
      "rewards/margins": 1.5491665601730347,
      "rewards/rejected": -2.5591635704040527,
      "step": 233
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.766769653642761,
      "kl": 0.0,
      "learning_rate": 4.387755102040816e-07,
      "logps/chosen": -183.52944946289062,
      "logps/rejected": -254.58596801757812,
      "loss": 0.1539,
      "rewards/chosen": -0.7764175534248352,
      "rewards/margins": 2.6363468170166016,
      "rewards/rejected": -3.412764310836792,
      "step": 234
    },
    {
      "epoch": 0.12,
      "grad_norm": 11.608047188492522,
      "kl": 0.0,
      "learning_rate": 4.3851386708529566e-07,
      "logps/chosen": -210.39288330078125,
      "logps/rejected": -248.83990478515625,
      "loss": 0.1721,
      "rewards/chosen": -0.28012871742248535,
      "rewards/margins": 2.2773919105529785,
      "rewards/rejected": -2.557520627975464,
      "step": 235
    },
    {
      "epoch": 0.12,
      "grad_norm": 13.662061510468588,
      "kl": 0.0,
      "learning_rate": 4.3825222396650966e-07,
      "logps/chosen": -257.81976318359375,
      "logps/rejected": -294.05328369140625,
      "loss": 0.1683,
      "rewards/chosen": 0.09654220193624496,
      "rewards/margins": 2.7226510047912598,
      "rewards/rejected": -2.6261088848114014,
      "step": 236
    },
    {
      "epoch": 0.12,
      "grad_norm": 12.161426046134066,
      "kl": 0.0,
      "learning_rate": 4.3799058084772366e-07,
      "logps/chosen": -248.7794647216797,
      "logps/rejected": -260.9866943359375,
      "loss": 0.1904,
      "rewards/chosen": 0.21598704159259796,
      "rewards/margins": 3.1339871883392334,
      "rewards/rejected": -2.9180002212524414,
      "step": 237
    },
    {
      "epoch": 0.12,
      "grad_norm": 10.031617578976944,
      "kl": 0.0,
      "learning_rate": 4.377289377289377e-07,
      "logps/chosen": -239.06961059570312,
      "logps/rejected": -262.04071044921875,
      "loss": 0.1708,
      "rewards/chosen": 0.007866961881518364,
      "rewards/margins": 2.7547590732574463,
      "rewards/rejected": -2.746892213821411,
      "step": 238
    },
    {
      "epoch": 0.13,
      "grad_norm": 14.122345962730769,
      "kl": 0.0,
      "learning_rate": 4.3746729461015176e-07,
      "logps/chosen": -192.8285369873047,
      "logps/rejected": -284.4400634765625,
      "loss": 0.2025,
      "rewards/chosen": -0.6923890709877014,
      "rewards/margins": 1.7750904560089111,
      "rewards/rejected": -2.4674794673919678,
      "step": 239
    },
    {
      "epoch": 0.13,
      "grad_norm": 10.12440657869901,
      "kl": 0.0,
      "learning_rate": 4.3720565149136575e-07,
      "logps/chosen": -254.7532958984375,
      "logps/rejected": -278.2186279296875,
      "loss": 0.2537,
      "rewards/chosen": -1.5660591125488281,
      "rewards/margins": 1.244922161102295,
      "rewards/rejected": -2.810981273651123,
      "step": 240
    },
    {
      "epoch": 0.13,
      "grad_norm": 9.097833834404545,
      "kl": 0.0,
      "learning_rate": 4.3694400837257975e-07,
      "logps/chosen": -190.60516357421875,
      "logps/rejected": -281.48699951171875,
      "loss": 0.1973,
      "rewards/chosen": -0.27278468012809753,
      "rewards/margins": 2.6037862300872803,
      "rewards/rejected": -2.8765709400177,
      "step": 241
    },
    {
      "epoch": 0.13,
      "grad_norm": 10.170278536494735,
      "kl": 0.0,
      "learning_rate": 4.366823652537938e-07,
      "logps/chosen": -241.40150451660156,
      "logps/rejected": -271.16192626953125,
      "loss": 0.1001,
      "rewards/chosen": 0.05600461736321449,
      "rewards/margins": 4.060368537902832,
      "rewards/rejected": -4.004364013671875,
      "step": 242
    },
    {
      "epoch": 0.13,
      "grad_norm": 10.968657094423396,
      "kl": 0.0,
      "learning_rate": 4.3642072213500785e-07,
      "logps/chosen": -218.7855987548828,
      "logps/rejected": -231.13519287109375,
      "loss": 0.1814,
      "rewards/chosen": 0.2664739489555359,
      "rewards/margins": 1.7813398838043213,
      "rewards/rejected": -1.5148659944534302,
      "step": 243
    },
    {
      "epoch": 0.13,
      "grad_norm": 8.862306842445157,
      "kl": 0.0,
      "learning_rate": 4.3615907901622185e-07,
      "logps/chosen": -195.7134552001953,
      "logps/rejected": -222.33502197265625,
      "loss": 0.096,
      "rewards/chosen": 0.3085781931877136,
      "rewards/margins": 2.4294745922088623,
      "rewards/rejected": -2.120896339416504,
      "step": 244
    },
    {
      "epoch": 0.13,
      "grad_norm": 10.488199983055214,
      "kl": 0.0,
      "learning_rate": 4.358974358974359e-07,
      "logps/chosen": -213.25262451171875,
      "logps/rejected": -245.7234344482422,
      "loss": 0.1646,
      "rewards/chosen": 0.47856229543685913,
      "rewards/margins": 4.145866870880127,
      "rewards/rejected": -3.667304754257202,
      "step": 245
    },
    {
      "epoch": 0.13,
      "grad_norm": 11.576912231416909,
      "kl": 0.0,
      "learning_rate": 4.3563579277864995e-07,
      "logps/chosen": -215.87538146972656,
      "logps/rejected": -262.62786865234375,
      "loss": 0.2232,
      "rewards/chosen": 0.43594250082969666,
      "rewards/margins": 1.7815001010894775,
      "rewards/rejected": -1.3455575704574585,
      "step": 246
    },
    {
      "epoch": 0.13,
      "grad_norm": 11.472489865502666,
      "kl": 0.0,
      "learning_rate": 4.353741496598639e-07,
      "logps/chosen": -298.4697570800781,
      "logps/rejected": -223.03550720214844,
      "loss": 0.1638,
      "rewards/chosen": 0.32377567887306213,
      "rewards/margins": 1.912637710571289,
      "rewards/rejected": -1.5888620615005493,
      "step": 247
    },
    {
      "epoch": 0.13,
      "grad_norm": 9.245318090630139,
      "kl": 0.0,
      "learning_rate": 4.3511250654107794e-07,
      "logps/chosen": -160.44625854492188,
      "logps/rejected": -268.469482421875,
      "loss": 0.1647,
      "rewards/chosen": 0.2935628294944763,
      "rewards/margins": 4.304611682891846,
      "rewards/rejected": -4.011048793792725,
      "step": 248
    },
    {
      "epoch": 0.13,
      "grad_norm": 9.203079637794323,
      "kl": 0.0,
      "learning_rate": 4.34850863422292e-07,
      "logps/chosen": -248.113037109375,
      "logps/rejected": -241.7339630126953,
      "loss": 0.1833,
      "rewards/chosen": 1.2330036163330078,
      "rewards/margins": 3.8498387336730957,
      "rewards/rejected": -2.616835117340088,
      "step": 249
    },
    {
      "epoch": 0.13,
      "grad_norm": 9.144257332584273,
      "kl": 0.0,
      "learning_rate": 4.34589220303506e-07,
      "logps/chosen": -196.73558044433594,
      "logps/rejected": -234.81517028808594,
      "loss": 0.19,
      "rewards/chosen": -0.7745031714439392,
      "rewards/margins": 1.9112095832824707,
      "rewards/rejected": -2.6857128143310547,
      "step": 250
    },
    {
      "epoch": 0.13,
      "grad_norm": 10.977345554295628,
      "kl": 0.0,
      "learning_rate": 4.3432757718472004e-07,
      "logps/chosen": -270.6989440917969,
      "logps/rejected": -296.6756896972656,
      "loss": 0.1977,
      "rewards/chosen": 0.5559626221656799,
      "rewards/margins": 2.0558934211730957,
      "rewards/rejected": -1.499930739402771,
      "step": 251
    },
    {
      "epoch": 0.13,
      "grad_norm": 11.195892622420404,
      "kl": 0.0,
      "learning_rate": 4.3406593406593404e-07,
      "logps/chosen": -324.94915771484375,
      "logps/rejected": -269.9296569824219,
      "loss": 0.1905,
      "rewards/chosen": -0.5299755334854126,
      "rewards/margins": 2.1768460273742676,
      "rewards/rejected": -2.7068214416503906,
      "step": 252
    },
    {
      "epoch": 0.13,
      "grad_norm": 13.09663182243734,
      "kl": 0.0,
      "learning_rate": 4.3380429094714804e-07,
      "logps/chosen": -209.63999938964844,
      "logps/rejected": -290.00927734375,
      "loss": 0.1509,
      "rewards/chosen": 0.9980623722076416,
      "rewards/margins": 4.123095512390137,
      "rewards/rejected": -3.125032901763916,
      "step": 253
    },
    {
      "epoch": 0.13,
      "grad_norm": 16.101792755673987,
      "kl": 0.0,
      "learning_rate": 4.335426478283621e-07,
      "logps/chosen": -178.9565887451172,
      "logps/rejected": -214.99896240234375,
      "loss": 0.1981,
      "rewards/chosen": -0.7419480085372925,
      "rewards/margins": 1.410202145576477,
      "rewards/rejected": -2.1521501541137695,
      "step": 254
    },
    {
      "epoch": 0.13,
      "grad_norm": 9.260673089248163,
      "kl": 0.0,
      "learning_rate": 4.3328100470957614e-07,
      "logps/chosen": -269.7500915527344,
      "logps/rejected": -240.69114685058594,
      "loss": 0.1547,
      "rewards/chosen": -1.514892578125,
      "rewards/margins": 1.8574752807617188,
      "rewards/rejected": -3.3723678588867188,
      "step": 255
    },
    {
      "epoch": 0.13,
      "grad_norm": 8.74207098415269,
      "kl": 0.0,
      "learning_rate": 4.330193615907902e-07,
      "logps/chosen": -238.84718322753906,
      "logps/rejected": -185.5977325439453,
      "loss": 0.2231,
      "rewards/chosen": 0.872572660446167,
      "rewards/margins": 2.413464069366455,
      "rewards/rejected": -1.5408915281295776,
      "step": 256
    },
    {
      "epoch": 0.13,
      "grad_norm": 13.06989888703404,
      "kl": 0.0,
      "learning_rate": 4.3275771847200413e-07,
      "logps/chosen": -198.09527587890625,
      "logps/rejected": -241.32310485839844,
      "loss": 0.293,
      "rewards/chosen": -0.5270448327064514,
      "rewards/margins": 1.5255239009857178,
      "rewards/rejected": -2.0525686740875244,
      "step": 257
    },
    {
      "epoch": 0.14,
      "grad_norm": 9.146455056162033,
      "kl": 0.0,
      "learning_rate": 4.324960753532182e-07,
      "logps/chosen": -158.8040771484375,
      "logps/rejected": -317.1865539550781,
      "loss": 0.1985,
      "rewards/chosen": -0.44321534037590027,
      "rewards/margins": 2.745103359222412,
      "rewards/rejected": -3.1883187294006348,
      "step": 258
    },
    {
      "epoch": 0.14,
      "grad_norm": 11.496122217797181,
      "kl": 0.0,
      "learning_rate": 4.3223443223443223e-07,
      "logps/chosen": -195.80255126953125,
      "logps/rejected": -251.7202911376953,
      "loss": 0.2603,
      "rewards/chosen": 0.14385934174060822,
      "rewards/margins": 3.09423565864563,
      "rewards/rejected": -2.950376272201538,
      "step": 259
    },
    {
      "epoch": 0.14,
      "grad_norm": 10.14234334654825,
      "kl": 0.0,
      "learning_rate": 4.3197278911564623e-07,
      "logps/chosen": -194.34381103515625,
      "logps/rejected": -210.70053100585938,
      "loss": 0.2247,
      "rewards/chosen": -0.5755199193954468,
      "rewards/margins": 1.121391773223877,
      "rewards/rejected": -1.6969116926193237,
      "step": 260
    },
    {
      "epoch": 0.14,
      "grad_norm": 10.972480345195148,
      "kl": 0.0,
      "learning_rate": 4.317111459968603e-07,
      "logps/chosen": -190.80508422851562,
      "logps/rejected": -250.46139526367188,
      "loss": 0.2264,
      "rewards/chosen": 0.17243629693984985,
      "rewards/margins": 3.771756887435913,
      "rewards/rejected": -3.599320650100708,
      "step": 261
    },
    {
      "epoch": 0.14,
      "grad_norm": 12.562599017075712,
      "kl": 0.0,
      "learning_rate": 4.314495028780743e-07,
      "logps/chosen": -251.84373474121094,
      "logps/rejected": -245.03099060058594,
      "loss": 0.1727,
      "rewards/chosen": 0.41944611072540283,
      "rewards/margins": 2.5281949043273926,
      "rewards/rejected": -2.1087489128112793,
      "step": 262
    },
    {
      "epoch": 0.14,
      "grad_norm": 11.382077716702321,
      "kl": 0.0,
      "learning_rate": 4.311878597592883e-07,
      "logps/chosen": -213.92518615722656,
      "logps/rejected": -231.60177612304688,
      "loss": 0.2323,
      "rewards/chosen": -0.6165014505386353,
      "rewards/margins": 1.244750738143921,
      "rewards/rejected": -1.8612521886825562,
      "step": 263
    },
    {
      "epoch": 0.14,
      "grad_norm": 9.731491692873679,
      "kl": 0.0,
      "learning_rate": 4.309262166405023e-07,
      "logps/chosen": -212.86976623535156,
      "logps/rejected": -261.46868896484375,
      "loss": 0.1888,
      "rewards/chosen": -0.4610190689563751,
      "rewards/margins": 2.1575005054473877,
      "rewards/rejected": -2.6185195446014404,
      "step": 264
    },
    {
      "epoch": 0.14,
      "grad_norm": 11.409608574249868,
      "kl": 0.0,
      "learning_rate": 4.306645735217164e-07,
      "logps/chosen": -224.6321258544922,
      "logps/rejected": -252.78160095214844,
      "loss": 0.2112,
      "rewards/chosen": 0.9657794833183289,
      "rewards/margins": 2.610295534133911,
      "rewards/rejected": -1.6445159912109375,
      "step": 265
    },
    {
      "epoch": 0.14,
      "grad_norm": 9.856888071447521,
      "kl": 0.0,
      "learning_rate": 4.3040293040293043e-07,
      "logps/chosen": -222.96458435058594,
      "logps/rejected": -214.23255920410156,
      "loss": 0.2125,
      "rewards/chosen": -0.6956588625907898,
      "rewards/margins": 1.7711460590362549,
      "rewards/rejected": -2.4668049812316895,
      "step": 266
    },
    {
      "epoch": 0.14,
      "grad_norm": 12.139684536745287,
      "kl": 0.0,
      "learning_rate": 4.3014128728414437e-07,
      "logps/chosen": -227.04371643066406,
      "logps/rejected": -338.61358642578125,
      "loss": 0.1782,
      "rewards/chosen": -0.1494772583246231,
      "rewards/margins": 2.7432305812835693,
      "rewards/rejected": -2.8927078247070312,
      "step": 267
    },
    {
      "epoch": 0.14,
      "grad_norm": 11.372427757605506,
      "kl": 0.0,
      "learning_rate": 4.298796441653584e-07,
      "logps/chosen": -269.498291015625,
      "logps/rejected": -285.0457763671875,
      "loss": 0.2566,
      "rewards/chosen": -0.7780652642250061,
      "rewards/margins": 2.5229103565216064,
      "rewards/rejected": -3.3009755611419678,
      "step": 268
    },
    {
      "epoch": 0.14,
      "grad_norm": 10.418607245045079,
      "kl": 0.0,
      "learning_rate": 4.2961800104657247e-07,
      "logps/chosen": -206.6187744140625,
      "logps/rejected": -178.76629638671875,
      "loss": 0.1973,
      "rewards/chosen": -0.2997088134288788,
      "rewards/margins": 2.2873473167419434,
      "rewards/rejected": -2.5870561599731445,
      "step": 269
    },
    {
      "epoch": 0.14,
      "grad_norm": 8.700783144862982,
      "kl": 0.0,
      "learning_rate": 4.2935635792778647e-07,
      "logps/chosen": -259.9353942871094,
      "logps/rejected": -190.39688110351562,
      "loss": 0.1152,
      "rewards/chosen": 0.6818735003471375,
      "rewards/margins": 2.6719298362731934,
      "rewards/rejected": -1.9900562763214111,
      "step": 270
    },
    {
      "epoch": 0.14,
      "grad_norm": 11.113524020838248,
      "kl": 0.0,
      "learning_rate": 4.290947148090005e-07,
      "logps/chosen": -278.58087158203125,
      "logps/rejected": -203.67617797851562,
      "loss": 0.1375,
      "rewards/chosen": 0.04530330374836922,
      "rewards/margins": 3.534538507461548,
      "rewards/rejected": -3.4892351627349854,
      "step": 271
    },
    {
      "epoch": 0.14,
      "grad_norm": 8.408405701936438,
      "kl": 0.0,
      "learning_rate": 4.288330716902145e-07,
      "logps/chosen": -167.8460693359375,
      "logps/rejected": -220.5739288330078,
      "loss": 0.1761,
      "rewards/chosen": -0.373342365026474,
      "rewards/margins": 2.532893657684326,
      "rewards/rejected": -2.906235933303833,
      "step": 272
    },
    {
      "epoch": 0.14,
      "grad_norm": 13.207340486836898,
      "kl": 0.0,
      "learning_rate": 4.285714285714285e-07,
      "logps/chosen": -271.5782165527344,
      "logps/rejected": -229.74559020996094,
      "loss": 0.1837,
      "rewards/chosen": -0.2260454148054123,
      "rewards/margins": 1.8243522644042969,
      "rewards/rejected": -2.0503976345062256,
      "step": 273
    },
    {
      "epoch": 0.14,
      "grad_norm": 9.722222876772891,
      "kl": 0.0,
      "learning_rate": 4.2830978545264256e-07,
      "logps/chosen": -263.2925109863281,
      "logps/rejected": -304.8329772949219,
      "loss": 0.1408,
      "rewards/chosen": -0.7855675220489502,
      "rewards/margins": 1.542686939239502,
      "rewards/rejected": -2.328254461288452,
      "step": 274
    },
    {
      "epoch": 0.14,
      "grad_norm": 11.917040600069175,
      "kl": 0.0,
      "learning_rate": 4.280481423338566e-07,
      "logps/chosen": -200.11573791503906,
      "logps/rejected": -282.0390930175781,
      "loss": 0.1633,
      "rewards/chosen": 0.374490350484848,
      "rewards/margins": 3.0798637866973877,
      "rewards/rejected": -2.705373525619507,
      "step": 275
    },
    {
      "epoch": 0.14,
      "grad_norm": 14.079400827374393,
      "kl": 0.0,
      "learning_rate": 4.2778649921507067e-07,
      "logps/chosen": -226.02078247070312,
      "logps/rejected": -285.05780029296875,
      "loss": 0.239,
      "rewards/chosen": 0.22204646468162537,
      "rewards/margins": 2.745478630065918,
      "rewards/rejected": -2.5234322547912598,
      "step": 276
    },
    {
      "epoch": 0.14,
      "grad_norm": 12.40231967594057,
      "kl": 0.0,
      "learning_rate": 4.2752485609628466e-07,
      "logps/chosen": -233.96995544433594,
      "logps/rejected": -245.3420867919922,
      "loss": 0.2526,
      "rewards/chosen": -0.20722751319408417,
      "rewards/margins": 1.8177690505981445,
      "rewards/rejected": -2.024996519088745,
      "step": 277
    },
    {
      "epoch": 0.15,
      "grad_norm": 11.912714615917263,
      "kl": 0.0,
      "learning_rate": 4.2726321297749866e-07,
      "logps/chosen": -242.40673828125,
      "logps/rejected": -333.7016906738281,
      "loss": 0.2233,
      "rewards/chosen": -0.6541883945465088,
      "rewards/margins": 1.2289773225784302,
      "rewards/rejected": -1.883165717124939,
      "step": 278
    },
    {
      "epoch": 0.15,
      "grad_norm": 12.620911514266433,
      "kl": 0.0,
      "learning_rate": 4.270015698587127e-07,
      "logps/chosen": -158.87258911132812,
      "logps/rejected": -180.81192016601562,
      "loss": 0.1446,
      "rewards/chosen": -0.11502157896757126,
      "rewards/margins": 1.9224528074264526,
      "rewards/rejected": -2.0374743938446045,
      "step": 279
    },
    {
      "epoch": 0.15,
      "grad_norm": 11.601791307438955,
      "kl": 0.0,
      "learning_rate": 4.267399267399267e-07,
      "logps/chosen": -245.42486572265625,
      "logps/rejected": -260.4696960449219,
      "loss": 0.1418,
      "rewards/chosen": 0.19518660008907318,
      "rewards/margins": 2.677379846572876,
      "rewards/rejected": -2.4821932315826416,
      "step": 280
    },
    {
      "epoch": 0.15,
      "grad_norm": 12.839705365539672,
      "kl": 0.0,
      "learning_rate": 4.2647828362114076e-07,
      "logps/chosen": -256.3037109375,
      "logps/rejected": -272.7105712890625,
      "loss": 0.2253,
      "rewards/chosen": 0.4001716375350952,
      "rewards/margins": 2.9852728843688965,
      "rewards/rejected": -2.585101366043091,
      "step": 281
    },
    {
      "epoch": 0.15,
      "grad_norm": 11.039693610954819,
      "kl": 0.0,
      "learning_rate": 4.262166405023548e-07,
      "logps/chosen": -256.00457763671875,
      "logps/rejected": -206.69845581054688,
      "loss": 0.1776,
      "rewards/chosen": 0.3543960452079773,
      "rewards/margins": 2.914128303527832,
      "rewards/rejected": -2.55973219871521,
      "step": 282
    },
    {
      "epoch": 0.15,
      "grad_norm": 9.759460114772729,
      "kl": 0.0,
      "learning_rate": 4.2595499738356875e-07,
      "logps/chosen": -273.74395751953125,
      "logps/rejected": -195.93484497070312,
      "loss": 0.2458,
      "rewards/chosen": 1.1952847242355347,
      "rewards/margins": 2.3585853576660156,
      "rewards/rejected": -1.1633005142211914,
      "step": 283
    },
    {
      "epoch": 0.15,
      "grad_norm": 10.249902760795546,
      "kl": 0.0,
      "learning_rate": 4.256933542647828e-07,
      "logps/chosen": -250.92709350585938,
      "logps/rejected": -251.54632568359375,
      "loss": 0.2445,
      "rewards/chosen": -1.2972968816757202,
      "rewards/margins": 1.1200555562973022,
      "rewards/rejected": -2.4173524379730225,
      "step": 284
    },
    {
      "epoch": 0.15,
      "grad_norm": 11.519889613409475,
      "kl": 0.0,
      "learning_rate": 4.2543171114599685e-07,
      "logps/chosen": -241.36473083496094,
      "logps/rejected": -217.71559143066406,
      "loss": 0.1655,
      "rewards/chosen": -0.08067867159843445,
      "rewards/margins": 2.2561678886413574,
      "rewards/rejected": -2.3368465900421143,
      "step": 285
    },
    {
      "epoch": 0.15,
      "grad_norm": 12.116227642012538,
      "kl": 0.0,
      "learning_rate": 4.2517006802721085e-07,
      "logps/chosen": -227.16310119628906,
      "logps/rejected": -254.51699829101562,
      "loss": 0.1795,
      "rewards/chosen": -0.5490508675575256,
      "rewards/margins": 2.050340175628662,
      "rewards/rejected": -2.599390983581543,
      "step": 286
    },
    {
      "epoch": 0.15,
      "grad_norm": 10.761723965602993,
      "kl": 0.0,
      "learning_rate": 4.249084249084249e-07,
      "logps/chosen": -213.15887451171875,
      "logps/rejected": -229.43199157714844,
      "loss": 0.2217,
      "rewards/chosen": -0.36798590421676636,
      "rewards/margins": 0.6998893618583679,
      "rewards/rejected": -1.0678752660751343,
      "step": 287
    },
    {
      "epoch": 0.15,
      "grad_norm": 9.96614506259357,
      "kl": 0.0,
      "learning_rate": 4.246467817896389e-07,
      "logps/chosen": -224.59458923339844,
      "logps/rejected": -168.9609375,
      "loss": 0.2067,
      "rewards/chosen": 0.40960344672203064,
      "rewards/margins": 1.400376319885254,
      "rewards/rejected": -0.9907728433609009,
      "step": 288
    },
    {
      "epoch": 0.15,
      "grad_norm": 10.590063389968323,
      "kl": 0.0,
      "learning_rate": 4.2438513867085295e-07,
      "logps/chosen": -230.34326171875,
      "logps/rejected": -194.3732452392578,
      "loss": 0.1615,
      "rewards/chosen": -0.03669742867350578,
      "rewards/margins": 1.9274276494979858,
      "rewards/rejected": -1.9641250371932983,
      "step": 289
    },
    {
      "epoch": 0.15,
      "grad_norm": 12.341220101671244,
      "kl": 0.0,
      "learning_rate": 4.2412349555206695e-07,
      "logps/chosen": -261.1521911621094,
      "logps/rejected": -184.7381591796875,
      "loss": 0.1907,
      "rewards/chosen": 0.5180913209915161,
      "rewards/margins": 1.7361607551574707,
      "rewards/rejected": -1.2180694341659546,
      "step": 290
    },
    {
      "epoch": 0.15,
      "grad_norm": 13.945755502648126,
      "kl": 0.0,
      "learning_rate": 4.23861852433281e-07,
      "logps/chosen": -318.291015625,
      "logps/rejected": -216.2500457763672,
      "loss": 0.2175,
      "rewards/chosen": -0.4215258061885834,
      "rewards/margins": 0.7515729665756226,
      "rewards/rejected": -1.1730988025665283,
      "step": 291
    },
    {
      "epoch": 0.15,
      "grad_norm": 11.888573752629188,
      "kl": 0.0,
      "learning_rate": 4.2360020931449505e-07,
      "logps/chosen": -213.46910095214844,
      "logps/rejected": -188.46728515625,
      "loss": 0.2024,
      "rewards/chosen": 0.058938559144735336,
      "rewards/margins": 1.171943187713623,
      "rewards/rejected": -1.1130046844482422,
      "step": 292
    },
    {
      "epoch": 0.15,
      "grad_norm": 16.17419414361992,
      "kl": 0.0,
      "learning_rate": 4.23338566195709e-07,
      "logps/chosen": -192.65357971191406,
      "logps/rejected": -261.0958251953125,
      "loss": 0.2653,
      "rewards/chosen": 0.42944175004959106,
      "rewards/margins": 1.398432970046997,
      "rewards/rejected": -0.9689911603927612,
      "step": 293
    },
    {
      "epoch": 0.15,
      "grad_norm": 9.577005409188526,
      "kl": 0.0,
      "learning_rate": 4.2307692307692304e-07,
      "logps/chosen": -214.690185546875,
      "logps/rejected": -195.5751953125,
      "loss": 0.1916,
      "rewards/chosen": 0.16525867581367493,
      "rewards/margins": 1.8478789329528809,
      "rewards/rejected": -1.6826202869415283,
      "step": 294
    },
    {
      "epoch": 0.15,
      "grad_norm": 13.024241787173485,
      "kl": 0.0,
      "learning_rate": 4.228152799581371e-07,
      "logps/chosen": -269.0843505859375,
      "logps/rejected": -258.8379821777344,
      "loss": 0.1626,
      "rewards/chosen": 0.0963350236415863,
      "rewards/margins": 1.255967617034912,
      "rewards/rejected": -1.1596325635910034,
      "step": 295
    },
    {
      "epoch": 0.15,
      "grad_norm": 8.85778497897942,
      "kl": 0.0,
      "learning_rate": 4.225536368393511e-07,
      "logps/chosen": -217.76644897460938,
      "logps/rejected": -309.886962890625,
      "loss": 0.112,
      "rewards/chosen": 0.8563817739486694,
      "rewards/margins": 3.093313217163086,
      "rewards/rejected": -2.236931562423706,
      "step": 296
    },
    {
      "epoch": 0.16,
      "grad_norm": 12.645142193880144,
      "kl": 0.0,
      "learning_rate": 4.2229199372056514e-07,
      "logps/chosen": -181.10629272460938,
      "logps/rejected": -232.3704071044922,
      "loss": 0.2186,
      "rewards/chosen": -0.1876239776611328,
      "rewards/margins": 1.843381643295288,
      "rewards/rejected": -2.031005620956421,
      "step": 297
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.248387892063585,
      "kl": 0.0,
      "learning_rate": 4.2203035060177914e-07,
      "logps/chosen": -248.957763671875,
      "logps/rejected": -215.41151428222656,
      "loss": 0.1798,
      "rewards/chosen": 0.26162558794021606,
      "rewards/margins": 2.0767014026641846,
      "rewards/rejected": -1.8150758743286133,
      "step": 298
    },
    {
      "epoch": 0.16,
      "grad_norm": 13.744936576539429,
      "kl": 0.0,
      "learning_rate": 4.217687074829932e-07,
      "logps/chosen": -221.40310668945312,
      "logps/rejected": -279.14703369140625,
      "loss": 0.2128,
      "rewards/chosen": 0.0988471731543541,
      "rewards/margins": 2.282918691635132,
      "rewards/rejected": -2.1840715408325195,
      "step": 299
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.139613060769499,
      "kl": 0.0,
      "learning_rate": 4.215070643642072e-07,
      "logps/chosen": -219.13638305664062,
      "logps/rejected": -316.4018249511719,
      "loss": 0.1851,
      "rewards/chosen": -0.3484467566013336,
      "rewards/margins": 2.9719343185424805,
      "rewards/rejected": -3.3203811645507812,
      "step": 300
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.23566454390853,
      "kl": 0.0,
      "learning_rate": 4.2124542124542123e-07,
      "logps/chosen": -264.99456787109375,
      "logps/rejected": -212.24009704589844,
      "loss": 0.2013,
      "rewards/chosen": 0.5730603337287903,
      "rewards/margins": 2.691094398498535,
      "rewards/rejected": -2.1180341243743896,
      "step": 301
    },
    {
      "epoch": 0.16,
      "grad_norm": 9.57651530765622,
      "kl": 0.0,
      "learning_rate": 4.209837781266353e-07,
      "logps/chosen": -220.54957580566406,
      "logps/rejected": -235.98797607421875,
      "loss": 0.1582,
      "rewards/chosen": 0.9783976674079895,
      "rewards/margins": 2.1505749225616455,
      "rewards/rejected": -1.1721771955490112,
      "step": 302
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.869078446374544,
      "kl": 0.0,
      "learning_rate": 4.2072213500784923e-07,
      "logps/chosen": -275.3968505859375,
      "logps/rejected": -187.9066925048828,
      "loss": 0.1583,
      "rewards/chosen": 0.454397588968277,
      "rewards/margins": 1.851892352104187,
      "rewards/rejected": -1.3974947929382324,
      "step": 303
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.706252453621756,
      "kl": 0.0,
      "learning_rate": 4.204604918890633e-07,
      "logps/chosen": -310.59661865234375,
      "logps/rejected": -275.7345275878906,
      "loss": 0.2664,
      "rewards/chosen": -0.8155102133750916,
      "rewards/margins": 0.8342878222465515,
      "rewards/rejected": -1.649798035621643,
      "step": 304
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.279199741590181,
      "kl": 0.0,
      "learning_rate": 4.2019884877027733e-07,
      "logps/chosen": -238.77163696289062,
      "logps/rejected": -267.1834716796875,
      "loss": 0.1667,
      "rewards/chosen": 0.08637741953134537,
      "rewards/margins": 2.7349460124969482,
      "rewards/rejected": -2.648568630218506,
      "step": 305
    },
    {
      "epoch": 0.16,
      "grad_norm": 10.974707777807534,
      "kl": 0.0,
      "learning_rate": 4.1993720565149133e-07,
      "logps/chosen": -249.7024383544922,
      "logps/rejected": -290.3708190917969,
      "loss": 0.2169,
      "rewards/chosen": 0.3819645047187805,
      "rewards/margins": 2.038438558578491,
      "rewards/rejected": -1.6564741134643555,
      "step": 306
    },
    {
      "epoch": 0.16,
      "grad_norm": 15.241423459831774,
      "kl": 0.0,
      "learning_rate": 4.196755625327054e-07,
      "logps/chosen": -256.07049560546875,
      "logps/rejected": -295.1194152832031,
      "loss": 0.1785,
      "rewards/chosen": 0.4329548180103302,
      "rewards/margins": 2.4334588050842285,
      "rewards/rejected": -2.0005040168762207,
      "step": 307
    },
    {
      "epoch": 0.16,
      "grad_norm": 12.073965597469332,
      "kl": 0.0,
      "learning_rate": 4.1941391941391943e-07,
      "logps/chosen": -233.57545471191406,
      "logps/rejected": -308.6473388671875,
      "loss": 0.1959,
      "rewards/chosen": -0.34189051389694214,
      "rewards/margins": 1.3108210563659668,
      "rewards/rejected": -1.6527115106582642,
      "step": 308
    },
    {
      "epoch": 0.16,
      "grad_norm": 10.835558830180704,
      "kl": 0.0,
      "learning_rate": 4.191522762951334e-07,
      "logps/chosen": -221.1939239501953,
      "logps/rejected": -216.98193359375,
      "loss": 0.1861,
      "rewards/chosen": 0.3302954435348511,
      "rewards/margins": 1.9516035318374634,
      "rewards/rejected": -1.6213080883026123,
      "step": 309
    },
    {
      "epoch": 0.16,
      "grad_norm": 10.5831850564,
      "kl": 0.0,
      "learning_rate": 4.188906331763474e-07,
      "logps/chosen": -193.68711853027344,
      "logps/rejected": -257.5124206542969,
      "loss": 0.2299,
      "rewards/chosen": -0.3544810116291046,
      "rewards/margins": 1.7205111980438232,
      "rewards/rejected": -2.0749921798706055,
      "step": 310
    },
    {
      "epoch": 0.16,
      "grad_norm": 12.791438944238683,
      "kl": 0.0,
      "learning_rate": 4.1862899005756147e-07,
      "logps/chosen": -259.3917236328125,
      "logps/rejected": -346.5878601074219,
      "loss": 0.2063,
      "rewards/chosen": -1.0072615146636963,
      "rewards/margins": 0.5092824697494507,
      "rewards/rejected": -1.516543984413147,
      "step": 311
    },
    {
      "epoch": 0.16,
      "grad_norm": 8.990082192323051,
      "kl": 0.0,
      "learning_rate": 4.183673469387755e-07,
      "logps/chosen": -185.4383544921875,
      "logps/rejected": -225.95294189453125,
      "loss": 0.1446,
      "rewards/chosen": 0.1784064769744873,
      "rewards/margins": 2.15664005279541,
      "rewards/rejected": -1.9782336950302124,
      "step": 312
    },
    {
      "epoch": 0.16,
      "grad_norm": 9.526272579899544,
      "kl": 0.0,
      "learning_rate": 4.181057038199895e-07,
      "logps/chosen": -220.56004333496094,
      "logps/rejected": -190.05809020996094,
      "loss": 0.1914,
      "rewards/chosen": 1.1748487949371338,
      "rewards/margins": 3.3340234756469727,
      "rewards/rejected": -2.159174680709839,
      "step": 313
    },
    {
      "epoch": 0.16,
      "grad_norm": 11.139585767174607,
      "kl": 0.0,
      "learning_rate": 4.178440607012035e-07,
      "logps/chosen": -268.95770263671875,
      "logps/rejected": -222.8634033203125,
      "loss": 0.2051,
      "rewards/chosen": -0.40160444378852844,
      "rewards/margins": 2.409646987915039,
      "rewards/rejected": -2.811251401901245,
      "step": 314
    },
    {
      "epoch": 0.16,
      "grad_norm": 10.949601505354943,
      "kl": 0.0,
      "learning_rate": 4.1758241758241757e-07,
      "logps/chosen": -186.237060546875,
      "logps/rejected": -245.27206420898438,
      "loss": 0.249,
      "rewards/chosen": -0.8131721019744873,
      "rewards/margins": 1.2247133255004883,
      "rewards/rejected": -2.0378854274749756,
      "step": 315
    },
    {
      "epoch": 0.17,
      "grad_norm": 11.351891085819371,
      "kl": 0.0,
      "learning_rate": 4.1732077446363157e-07,
      "logps/chosen": -201.30223083496094,
      "logps/rejected": -243.75341796875,
      "loss": 0.1895,
      "rewards/chosen": 0.3100842535495758,
      "rewards/margins": 2.5161705017089844,
      "rewards/rejected": -2.2060861587524414,
      "step": 316
    },
    {
      "epoch": 0.17,
      "grad_norm": 12.221976691002848,
      "kl": 0.0,
      "learning_rate": 4.170591313448456e-07,
      "logps/chosen": -232.7662353515625,
      "logps/rejected": -362.24041748046875,
      "loss": 0.1972,
      "rewards/chosen": 0.6142377853393555,
      "rewards/margins": 3.4600791931152344,
      "rewards/rejected": -2.845841407775879,
      "step": 317
    },
    {
      "epoch": 0.17,
      "grad_norm": 10.754382204616256,
      "kl": 0.0,
      "learning_rate": 4.1679748822605967e-07,
      "logps/chosen": -215.42227172851562,
      "logps/rejected": -278.8962707519531,
      "loss": 0.1188,
      "rewards/chosen": 0.1050441712141037,
      "rewards/margins": 3.3125483989715576,
      "rewards/rejected": -3.2075042724609375,
      "step": 318
    },
    {
      "epoch": 0.17,
      "grad_norm": 12.987540188284319,
      "kl": 0.0,
      "learning_rate": 4.1653584510727366e-07,
      "logps/chosen": -244.31643676757812,
      "logps/rejected": -323.1125183105469,
      "loss": 0.2359,
      "rewards/chosen": -0.4736923575401306,
      "rewards/margins": 2.1419007778167725,
      "rewards/rejected": -2.615593194961548,
      "step": 319
    },
    {
      "epoch": 0.17,
      "grad_norm": 8.80349986892105,
      "kl": 0.0,
      "learning_rate": 4.1627420198848766e-07,
      "logps/chosen": -223.25929260253906,
      "logps/rejected": -236.41934204101562,
      "loss": 0.1387,
      "rewards/chosen": -0.46407273411750793,
      "rewards/margins": 1.7442665100097656,
      "rewards/rejected": -2.208339214324951,
      "step": 320
    },
    {
      "epoch": 0.17,
      "grad_norm": 13.1846568437176,
      "kl": 0.0,
      "learning_rate": 4.160125588697017e-07,
      "logps/chosen": -213.67198181152344,
      "logps/rejected": -295.5606384277344,
      "loss": 0.2656,
      "rewards/chosen": -0.33450353145599365,
      "rewards/margins": 1.9383219480514526,
      "rewards/rejected": -2.2728254795074463,
      "step": 321
    },
    {
      "epoch": 0.17,
      "grad_norm": 13.253044198831535,
      "kl": 0.0,
      "learning_rate": 4.1575091575091576e-07,
      "logps/chosen": -231.430908203125,
      "logps/rejected": -249.1555938720703,
      "loss": 0.1841,
      "rewards/chosen": 0.48544788360595703,
      "rewards/margins": 1.7013275623321533,
      "rewards/rejected": -1.2158796787261963,
      "step": 322
    },
    {
      "epoch": 0.17,
      "grad_norm": 11.874190668515645,
      "kl": 0.0,
      "learning_rate": 4.1548927263212976e-07,
      "logps/chosen": -208.916015625,
      "logps/rejected": -248.04067993164062,
      "loss": 0.1922,
      "rewards/chosen": -0.29332485795021057,
      "rewards/margins": 2.6810503005981445,
      "rewards/rejected": -2.9743752479553223,
      "step": 323
    },
    {
      "epoch": 0.17,
      "grad_norm": 9.788377245433486,
      "kl": 0.0,
      "learning_rate": 4.1522762951334376e-07,
      "logps/chosen": -237.29441833496094,
      "logps/rejected": -246.01272583007812,
      "loss": 0.2483,
      "rewards/chosen": -1.0158889293670654,
      "rewards/margins": 2.2013275623321533,
      "rewards/rejected": -3.2172164916992188,
      "step": 324
    },
    {
      "epoch": 0.17,
      "grad_norm": 10.674006139111874,
      "kl": 0.0,
      "learning_rate": 4.149659863945578e-07,
      "logps/chosen": -240.6201171875,
      "logps/rejected": -255.26345825195312,
      "loss": 0.2101,
      "rewards/chosen": -0.13020078837871552,
      "rewards/margins": 3.523922920227051,
      "rewards/rejected": -3.6541237831115723,
      "step": 325
    },
    {
      "epoch": 0.17,
      "grad_norm": 10.0670473259245,
      "kl": 0.0,
      "learning_rate": 4.147043432757718e-07,
      "logps/chosen": -229.05381774902344,
      "logps/rejected": -261.8768005371094,
      "loss": 0.1158,
      "rewards/chosen": -0.555793821811676,
      "rewards/margins": 3.118558645248413,
      "rewards/rejected": -3.6743524074554443,
      "step": 326
    },
    {
      "epoch": 0.17,
      "grad_norm": 8.521982319858306,
      "kl": 0.0,
      "learning_rate": 4.1444270015698585e-07,
      "logps/chosen": -220.83934020996094,
      "logps/rejected": -249.3174285888672,
      "loss": 0.1808,
      "rewards/chosen": 0.1714603751897812,
      "rewards/margins": 2.402540922164917,
      "rewards/rejected": -2.2310805320739746,
      "step": 327
    },
    {
      "epoch": 0.17,
      "grad_norm": 12.228304372001576,
      "kl": 0.0,
      "learning_rate": 4.141810570381999e-07,
      "logps/chosen": -286.3045654296875,
      "logps/rejected": -269.97601318359375,
      "loss": 0.1993,
      "rewards/chosen": -0.551383912563324,
      "rewards/margins": 1.8383245468139648,
      "rewards/rejected": -2.3897085189819336,
      "step": 328
    },
    {
      "epoch": 0.17,
      "grad_norm": 11.897560403128791,
      "kl": 0.0,
      "learning_rate": 4.1391941391941385e-07,
      "logps/chosen": -154.19729614257812,
      "logps/rejected": -238.20199584960938,
      "loss": 0.2342,
      "rewards/chosen": 0.1875210553407669,
      "rewards/margins": 1.9252369403839111,
      "rewards/rejected": -1.7377158403396606,
      "step": 329
    },
    {
      "epoch": 0.17,
      "grad_norm": 10.436673403488303,
      "kl": 0.0,
      "learning_rate": 4.136577708006279e-07,
      "logps/chosen": -209.2281036376953,
      "logps/rejected": -269.2527160644531,
      "loss": 0.1088,
      "rewards/chosen": 0.6344658732414246,
      "rewards/margins": 3.8878352642059326,
      "rewards/rejected": -3.2533693313598633,
      "step": 330
    },
    {
      "epoch": 0.17,
      "grad_norm": 8.255234754549457,
      "kl": 0.0,
      "learning_rate": 4.1339612768184195e-07,
      "logps/chosen": -212.30836486816406,
      "logps/rejected": -201.14207458496094,
      "loss": 0.174,
      "rewards/chosen": 0.6615203619003296,
      "rewards/margins": 2.292316436767578,
      "rewards/rejected": -1.630796194076538,
      "step": 331
    },
    {
      "epoch": 0.17,
      "grad_norm": 11.884943779747632,
      "kl": 0.0,
      "learning_rate": 4.13134484563056e-07,
      "logps/chosen": -211.07350158691406,
      "logps/rejected": -211.02023315429688,
      "loss": 0.2495,
      "rewards/chosen": 0.40837398171424866,
      "rewards/margins": 2.4228506088256836,
      "rewards/rejected": -2.0144765377044678,
      "step": 332
    },
    {
      "epoch": 0.17,
      "grad_norm": 11.994293665472632,
      "kl": 0.0,
      "learning_rate": 4.1287284144427e-07,
      "logps/chosen": -289.47589111328125,
      "logps/rejected": -228.80950927734375,
      "loss": 0.1505,
      "rewards/chosen": -0.011884563602507114,
      "rewards/margins": 1.357936143875122,
      "rewards/rejected": -1.3698207139968872,
      "step": 333
    },
    {
      "epoch": 0.17,
      "grad_norm": 12.497465243458791,
      "kl": 0.0,
      "learning_rate": 4.1261119832548405e-07,
      "logps/chosen": -214.80462646484375,
      "logps/rejected": -285.0194396972656,
      "loss": 0.2098,
      "rewards/chosen": -0.4318108856678009,
      "rewards/margins": 1.8095088005065918,
      "rewards/rejected": -2.2413196563720703,
      "step": 334
    },
    {
      "epoch": 0.18,
      "grad_norm": 12.597201422275237,
      "kl": 0.0,
      "learning_rate": 4.1234955520669805e-07,
      "logps/chosen": -173.9023895263672,
      "logps/rejected": -242.03558349609375,
      "loss": 0.1763,
      "rewards/chosen": -0.03515807166695595,
      "rewards/margins": 1.9462581872940063,
      "rewards/rejected": -1.9814162254333496,
      "step": 335
    },
    {
      "epoch": 0.18,
      "grad_norm": 11.29563689087554,
      "kl": 0.0,
      "learning_rate": 4.1208791208791204e-07,
      "logps/chosen": -224.08148193359375,
      "logps/rejected": -253.4733123779297,
      "loss": 0.1798,
      "rewards/chosen": 0.09991315752267838,
      "rewards/margins": 2.7761764526367188,
      "rewards/rejected": -2.6762633323669434,
      "step": 336
    },
    {
      "epoch": 0.18,
      "grad_norm": 9.798183455958176,
      "kl": 0.0,
      "learning_rate": 4.118262689691261e-07,
      "logps/chosen": -226.20651245117188,
      "logps/rejected": -261.2646789550781,
      "loss": 0.2247,
      "rewards/chosen": -0.36128562688827515,
      "rewards/margins": 2.0138986110687256,
      "rewards/rejected": -2.3751842975616455,
      "step": 337
    },
    {
      "epoch": 0.18,
      "grad_norm": 10.59217786675228,
      "kl": 0.0,
      "learning_rate": 4.1156462585034014e-07,
      "logps/chosen": -191.44393920898438,
      "logps/rejected": -222.71336364746094,
      "loss": 0.2218,
      "rewards/chosen": -0.10784044116735458,
      "rewards/margins": 1.7394548654556274,
      "rewards/rejected": -1.8472952842712402,
      "step": 338
    },
    {
      "epoch": 0.18,
      "grad_norm": 9.929264582178304,
      "kl": 0.0,
      "learning_rate": 4.1130298273155414e-07,
      "logps/chosen": -211.10133361816406,
      "logps/rejected": -235.98374938964844,
      "loss": 0.1557,
      "rewards/chosen": -0.17055515944957733,
      "rewards/margins": 3.654494285583496,
      "rewards/rejected": -3.82504940032959,
      "step": 339
    },
    {
      "epoch": 0.18,
      "grad_norm": 12.576617441384204,
      "kl": 0.0,
      "learning_rate": 4.1104133961276814e-07,
      "logps/chosen": -245.1005096435547,
      "logps/rejected": -265.1275939941406,
      "loss": 0.1974,
      "rewards/chosen": 0.29450762271881104,
      "rewards/margins": 1.8650548458099365,
      "rewards/rejected": -1.5705472230911255,
      "step": 340
    },
    {
      "epoch": 0.18,
      "grad_norm": 12.893258406638443,
      "kl": 0.0,
      "learning_rate": 4.107796964939822e-07,
      "logps/chosen": -283.796630859375,
      "logps/rejected": -255.04974365234375,
      "loss": 0.1138,
      "rewards/chosen": 1.2097806930541992,
      "rewards/margins": 3.861506223678589,
      "rewards/rejected": -2.6517255306243896,
      "step": 341
    },
    {
      "epoch": 0.18,
      "grad_norm": 9.442037096083547,
      "kl": 0.0,
      "learning_rate": 4.1051805337519624e-07,
      "logps/chosen": -193.82659912109375,
      "logps/rejected": -236.00262451171875,
      "loss": 0.169,
      "rewards/chosen": 0.25398653745651245,
      "rewards/margins": 3.0244903564453125,
      "rewards/rejected": -2.7705037593841553,
      "step": 342
    },
    {
      "epoch": 0.18,
      "grad_norm": 10.498722183093273,
      "kl": 0.0,
      "learning_rate": 4.1025641025641024e-07,
      "logps/chosen": -337.9830017089844,
      "logps/rejected": -247.3750457763672,
      "loss": 0.1899,
      "rewards/chosen": -1.3225940465927124,
      "rewards/margins": 1.0586594343185425,
      "rewards/rejected": -2.381253480911255,
      "step": 343
    },
    {
      "epoch": 0.18,
      "grad_norm": 10.172536259980642,
      "kl": 0.0,
      "learning_rate": 4.099947671376243e-07,
      "logps/chosen": -250.67410278320312,
      "logps/rejected": -271.3428039550781,
      "loss": 0.2109,
      "rewards/chosen": -0.8246206045150757,
      "rewards/margins": 2.084084987640381,
      "rewards/rejected": -2.908705472946167,
      "step": 344
    },
    {
      "epoch": 0.18,
      "grad_norm": 12.383051195247807,
      "kl": 0.0,
      "learning_rate": 4.097331240188383e-07,
      "logps/chosen": -263.4064025878906,
      "logps/rejected": -282.1274108886719,
      "loss": 0.2069,
      "rewards/chosen": -0.49988675117492676,
      "rewards/margins": 1.9791779518127441,
      "rewards/rejected": -2.479064702987671,
      "step": 345
    },
    {
      "epoch": 0.18,
      "grad_norm": 11.923267130594034,
      "kl": 0.0,
      "learning_rate": 4.094714809000523e-07,
      "logps/chosen": -215.8848419189453,
      "logps/rejected": -253.63076782226562,
      "loss": 0.1745,
      "rewards/chosen": -0.4700072109699249,
      "rewards/margins": 1.989243507385254,
      "rewards/rejected": -2.4592506885528564,
      "step": 346
    },
    {
      "epoch": 0.18,
      "grad_norm": 8.982041772561747,
      "kl": 0.0,
      "learning_rate": 4.0920983778126633e-07,
      "logps/chosen": -225.7939910888672,
      "logps/rejected": -244.41358947753906,
      "loss": 0.1869,
      "rewards/chosen": -1.364567756652832,
      "rewards/margins": 0.951836109161377,
      "rewards/rejected": -2.316403865814209,
      "step": 347
    },
    {
      "epoch": 0.18,
      "grad_norm": 10.837403601758242,
      "kl": 0.0,
      "learning_rate": 4.089481946624804e-07,
      "logps/chosen": -249.8412628173828,
      "logps/rejected": -241.59420776367188,
      "loss": 0.1471,
      "rewards/chosen": -0.19725684821605682,
      "rewards/margins": 2.119018077850342,
      "rewards/rejected": -2.316274881362915,
      "step": 348
    },
    {
      "epoch": 0.18,
      "grad_norm": 11.217732507310354,
      "kl": 0.0,
      "learning_rate": 4.086865515436944e-07,
      "logps/chosen": -257.2194519042969,
      "logps/rejected": -180.22837829589844,
      "loss": 0.1527,
      "rewards/chosen": 0.24126391112804413,
      "rewards/margins": 2.3532466888427734,
      "rewards/rejected": -2.111982822418213,
      "step": 349
    },
    {
      "epoch": 0.18,
      "grad_norm": 10.502546710908188,
      "kl": 0.0,
      "learning_rate": 4.084249084249084e-07,
      "logps/chosen": -227.2938232421875,
      "logps/rejected": -300.4139709472656,
      "loss": 0.1961,
      "rewards/chosen": -0.11248449236154556,
      "rewards/margins": 2.296675205230713,
      "rewards/rejected": -2.4091596603393555,
      "step": 350
    },
    {
      "epoch": 0.18,
      "grad_norm": 11.149043285356687,
      "kl": 0.0,
      "learning_rate": 4.0816326530612243e-07,
      "logps/chosen": -235.18333435058594,
      "logps/rejected": -206.1359100341797,
      "loss": 0.1829,
      "rewards/chosen": 0.5669969320297241,
      "rewards/margins": 3.09303617477417,
      "rewards/rejected": -2.5260391235351562,
      "step": 351
    },
    {
      "epoch": 0.18,
      "grad_norm": 11.899081712479965,
      "kl": 0.0,
      "learning_rate": 4.079016221873365e-07,
      "logps/chosen": -247.81387329101562,
      "logps/rejected": -268.5088195800781,
      "loss": 0.2918,
      "rewards/chosen": -0.36709415912628174,
      "rewards/margins": 0.9215909242630005,
      "rewards/rejected": -1.2886850833892822,
      "step": 352
    },
    {
      "epoch": 0.18,
      "grad_norm": 11.107384586901443,
      "kl": 0.0,
      "learning_rate": 4.076399790685505e-07,
      "logps/chosen": -206.99432373046875,
      "logps/rejected": -212.78196716308594,
      "loss": 0.2387,
      "rewards/chosen": -0.985337495803833,
      "rewards/margins": 1.2888743877410889,
      "rewards/rejected": -2.274211883544922,
      "step": 353
    },
    {
      "epoch": 0.19,
      "grad_norm": 10.235471285138322,
      "kl": 0.0,
      "learning_rate": 4.073783359497645e-07,
      "logps/chosen": -236.5115203857422,
      "logps/rejected": -272.5962219238281,
      "loss": 0.1737,
      "rewards/chosen": -0.7294275164604187,
      "rewards/margins": 2.0560388565063477,
      "rewards/rejected": -2.785466432571411,
      "step": 354
    },
    {
      "epoch": 0.19,
      "grad_norm": 11.272141410111164,
      "kl": 0.0,
      "learning_rate": 4.071166928309785e-07,
      "logps/chosen": -222.7964324951172,
      "logps/rejected": -287.06475830078125,
      "loss": 0.1308,
      "rewards/chosen": 0.3911401629447937,
      "rewards/margins": 2.989321231842041,
      "rewards/rejected": -2.5981810092926025,
      "step": 355
    },
    {
      "epoch": 0.19,
      "grad_norm": 7.964274506087736,
      "kl": 0.0,
      "learning_rate": 4.068550497121925e-07,
      "logps/chosen": -170.38925170898438,
      "logps/rejected": -205.2819366455078,
      "loss": 0.1071,
      "rewards/chosen": -0.18005786836147308,
      "rewards/margins": 2.6861469745635986,
      "rewards/rejected": -2.8662047386169434,
      "step": 356
    },
    {
      "epoch": 0.19,
      "grad_norm": 11.501254991289231,
      "kl": 0.0,
      "learning_rate": 4.0659340659340657e-07,
      "logps/chosen": -246.70559692382812,
      "logps/rejected": -238.14466857910156,
      "loss": 0.1549,
      "rewards/chosen": 0.33830830454826355,
      "rewards/margins": 2.2392632961273193,
      "rewards/rejected": -1.9009549617767334,
      "step": 357
    },
    {
      "epoch": 0.19,
      "grad_norm": 10.23573665557948,
      "kl": 0.0,
      "learning_rate": 4.063317634746206e-07,
      "logps/chosen": -200.4901123046875,
      "logps/rejected": -207.58871459960938,
      "loss": 0.1599,
      "rewards/chosen": 1.0193722248077393,
      "rewards/margins": 2.878417491912842,
      "rewards/rejected": -1.859045147895813,
      "step": 358
    },
    {
      "epoch": 0.19,
      "grad_norm": 11.84162361137601,
      "kl": 0.0,
      "learning_rate": 4.060701203558346e-07,
      "logps/chosen": -253.09637451171875,
      "logps/rejected": -247.7617645263672,
      "loss": 0.2949,
      "rewards/chosen": -1.3407307863235474,
      "rewards/margins": 0.6386681795120239,
      "rewards/rejected": -1.9793989658355713,
      "step": 359
    },
    {
      "epoch": 0.19,
      "grad_norm": 12.38802746375023,
      "kl": 0.0,
      "learning_rate": 4.058084772370486e-07,
      "logps/chosen": -241.19613647460938,
      "logps/rejected": -237.13682556152344,
      "loss": 0.1108,
      "rewards/chosen": -0.31301483511924744,
      "rewards/margins": 2.1045053005218506,
      "rewards/rejected": -2.417520046234131,
      "step": 360
    },
    {
      "epoch": 0.19,
      "grad_norm": 12.348300253030986,
      "kl": 0.0,
      "learning_rate": 4.0554683411826267e-07,
      "logps/chosen": -253.8113555908203,
      "logps/rejected": -250.69007873535156,
      "loss": 0.2231,
      "rewards/chosen": -0.6661314964294434,
      "rewards/margins": 1.135571002960205,
      "rewards/rejected": -1.8017024993896484,
      "step": 361
    },
    {
      "epoch": 0.19,
      "grad_norm": 9.813621316611632,
      "kl": 0.0,
      "learning_rate": 4.052851909994767e-07,
      "logps/chosen": -183.4186248779297,
      "logps/rejected": -226.8902130126953,
      "loss": 0.2452,
      "rewards/chosen": 0.09793578833341599,
      "rewards/margins": 3.1266567707061768,
      "rewards/rejected": -3.0287210941314697,
      "step": 362
    },
    {
      "epoch": 0.19,
      "grad_norm": 10.410184074704667,
      "kl": 0.0,
      "learning_rate": 4.050235478806907e-07,
      "logps/chosen": -218.2381134033203,
      "logps/rejected": -223.76805114746094,
      "loss": 0.2223,
      "rewards/chosen": -0.6022669672966003,
      "rewards/margins": 2.0103299617767334,
      "rewards/rejected": -2.6125969886779785,
      "step": 363
    },
    {
      "epoch": 0.19,
      "grad_norm": 12.01700363931878,
      "kl": 0.0,
      "learning_rate": 4.0476190476190476e-07,
      "logps/chosen": -232.75802612304688,
      "logps/rejected": -237.35382080078125,
      "loss": 0.2363,
      "rewards/chosen": 0.516566812992096,
      "rewards/margins": 2.019991874694824,
      "rewards/rejected": -1.5034250020980835,
      "step": 364
    },
    {
      "epoch": 0.19,
      "grad_norm": 10.262930905107156,
      "kl": 0.0,
      "learning_rate": 4.045002616431188e-07,
      "logps/chosen": -204.19009399414062,
      "logps/rejected": -235.92129516601562,
      "loss": 0.1872,
      "rewards/chosen": 0.05859723314642906,
      "rewards/margins": 2.741896629333496,
      "rewards/rejected": -2.6832993030548096,
      "step": 365
    },
    {
      "epoch": 0.19,
      "grad_norm": 11.30426818476053,
      "kl": 0.0,
      "learning_rate": 4.0423861852433276e-07,
      "logps/chosen": -166.32923889160156,
      "logps/rejected": -281.9306945800781,
      "loss": 0.2366,
      "rewards/chosen": -0.21419693529605865,
      "rewards/margins": 2.5291309356689453,
      "rewards/rejected": -2.7433278560638428,
      "step": 366
    },
    {
      "epoch": 0.19,
      "grad_norm": 10.791375928218388,
      "kl": 0.0,
      "learning_rate": 4.039769754055468e-07,
      "logps/chosen": -174.7226104736328,
      "logps/rejected": -268.91180419921875,
      "loss": 0.1773,
      "rewards/chosen": 0.03391220420598984,
      "rewards/margins": 3.911714792251587,
      "rewards/rejected": -3.877802610397339,
      "step": 367
    },
    {
      "epoch": 0.19,
      "grad_norm": 12.229778476365636,
      "kl": 0.0,
      "learning_rate": 4.0371533228676086e-07,
      "logps/chosen": -227.41676330566406,
      "logps/rejected": -253.11061096191406,
      "loss": 0.2316,
      "rewards/chosen": -1.3165072202682495,
      "rewards/margins": 2.440537452697754,
      "rewards/rejected": -3.757044553756714,
      "step": 368
    },
    {
      "epoch": 0.19,
      "grad_norm": 9.70449239918135,
      "kl": 0.0,
      "learning_rate": 4.0345368916797486e-07,
      "logps/chosen": -206.78543090820312,
      "logps/rejected": -193.22666931152344,
      "loss": 0.152,
      "rewards/chosen": -0.005291328299790621,
      "rewards/margins": 2.320930004119873,
      "rewards/rejected": -2.326221227645874,
      "step": 369
    },
    {
      "epoch": 0.19,
      "grad_norm": 12.529777714925808,
      "kl": 0.0,
      "learning_rate": 4.031920460491889e-07,
      "logps/chosen": -260.2597961425781,
      "logps/rejected": -245.1582794189453,
      "loss": 0.2229,
      "rewards/chosen": -0.34088611602783203,
      "rewards/margins": 2.3002560138702393,
      "rewards/rejected": -2.6411421298980713,
      "step": 370
    },
    {
      "epoch": 0.19,
      "grad_norm": 12.126319059662272,
      "kl": 0.0,
      "learning_rate": 4.029304029304029e-07,
      "logps/chosen": -159.91371154785156,
      "logps/rejected": -215.52037048339844,
      "loss": 0.1604,
      "rewards/chosen": -0.13704350590705872,
      "rewards/margins": 2.6966662406921387,
      "rewards/rejected": -2.833709716796875,
      "step": 371
    },
    {
      "epoch": 0.19,
      "grad_norm": 12.2279596400823,
      "kl": 0.0,
      "learning_rate": 4.026687598116169e-07,
      "logps/chosen": -203.8690643310547,
      "logps/rejected": -248.30355834960938,
      "loss": 0.2764,
      "rewards/chosen": -1.4684327840805054,
      "rewards/margins": 0.25741446018218994,
      "rewards/rejected": -1.7258472442626953,
      "step": 372
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.93017592019336,
      "kl": 0.0,
      "learning_rate": 4.0240711669283095e-07,
      "logps/chosen": -312.0763244628906,
      "logps/rejected": -262.0274963378906,
      "loss": 0.1746,
      "rewards/chosen": -1.0104618072509766,
      "rewards/margins": 2.119619369506836,
      "rewards/rejected": -3.1300811767578125,
      "step": 373
    },
    {
      "epoch": 0.2,
      "grad_norm": 10.026240675416432,
      "kl": 0.0,
      "learning_rate": 4.02145473574045e-07,
      "logps/chosen": -220.4754638671875,
      "logps/rejected": -265.4399719238281,
      "loss": 0.1718,
      "rewards/chosen": 0.43962326645851135,
      "rewards/margins": 3.062368392944336,
      "rewards/rejected": -2.6227450370788574,
      "step": 374
    },
    {
      "epoch": 0.2,
      "grad_norm": 7.862119363862361,
      "kl": 0.0,
      "learning_rate": 4.0188383045525905e-07,
      "logps/chosen": -225.05450439453125,
      "logps/rejected": -323.68798828125,
      "loss": 0.2182,
      "rewards/chosen": -0.6643018126487732,
      "rewards/margins": 1.284275770187378,
      "rewards/rejected": -1.9485775232315063,
      "step": 375
    },
    {
      "epoch": 0.2,
      "grad_norm": 10.113068546407243,
      "kl": 0.0,
      "learning_rate": 4.01622187336473e-07,
      "logps/chosen": -238.6414031982422,
      "logps/rejected": -226.67205810546875,
      "loss": 0.1479,
      "rewards/chosen": 0.5626780986785889,
      "rewards/margins": 3.329695224761963,
      "rewards/rejected": -2.767017126083374,
      "step": 376
    },
    {
      "epoch": 0.2,
      "grad_norm": 8.742780985362685,
      "kl": 0.0,
      "learning_rate": 4.0136054421768705e-07,
      "logps/chosen": -205.3953857421875,
      "logps/rejected": -243.7183074951172,
      "loss": 0.1611,
      "rewards/chosen": 0.14700005948543549,
      "rewards/margins": 3.2417192459106445,
      "rewards/rejected": -3.094719171524048,
      "step": 377
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.28132011147294,
      "kl": 0.0,
      "learning_rate": 4.010989010989011e-07,
      "logps/chosen": -214.97244262695312,
      "logps/rejected": -216.71173095703125,
      "loss": 0.1858,
      "rewards/chosen": -0.015776043757796288,
      "rewards/margins": 2.4526312351226807,
      "rewards/rejected": -2.468407392501831,
      "step": 378
    },
    {
      "epoch": 0.2,
      "grad_norm": 12.778638599311456,
      "kl": 0.0,
      "learning_rate": 4.008372579801151e-07,
      "logps/chosen": -281.7500915527344,
      "logps/rejected": -205.56297302246094,
      "loss": 0.1866,
      "rewards/chosen": -0.7977088093757629,
      "rewards/margins": 2.43152117729187,
      "rewards/rejected": -3.2292299270629883,
      "step": 379
    },
    {
      "epoch": 0.2,
      "grad_norm": 13.038247255191017,
      "kl": 0.0,
      "learning_rate": 4.0057561486132915e-07,
      "logps/chosen": -278.4736328125,
      "logps/rejected": -237.22613525390625,
      "loss": 0.172,
      "rewards/chosen": 0.31080275774002075,
      "rewards/margins": 2.6367199420928955,
      "rewards/rejected": -2.3259172439575195,
      "step": 380
    },
    {
      "epoch": 0.2,
      "grad_norm": 12.071589262477481,
      "kl": 0.0,
      "learning_rate": 4.0031397174254314e-07,
      "logps/chosen": -236.46629333496094,
      "logps/rejected": -230.3934326171875,
      "loss": 0.2205,
      "rewards/chosen": 0.46186843514442444,
      "rewards/margins": 2.98122239112854,
      "rewards/rejected": -2.5193538665771484,
      "step": 381
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.849099983442954,
      "kl": 0.0,
      "learning_rate": 4.0005232862375714e-07,
      "logps/chosen": -225.62376403808594,
      "logps/rejected": -240.21817016601562,
      "loss": 0.2767,
      "rewards/chosen": -0.6141564846038818,
      "rewards/margins": 0.870281457901001,
      "rewards/rejected": -1.4844379425048828,
      "step": 382
    },
    {
      "epoch": 0.2,
      "grad_norm": 10.00253243924632,
      "kl": 0.0,
      "learning_rate": 3.997906855049712e-07,
      "logps/chosen": -253.96109008789062,
      "logps/rejected": -244.46368408203125,
      "loss": 0.1898,
      "rewards/chosen": -0.6670475006103516,
      "rewards/margins": 2.5981571674346924,
      "rewards/rejected": -3.265204668045044,
      "step": 383
    },
    {
      "epoch": 0.2,
      "grad_norm": 9.423555936016577,
      "kl": 0.0,
      "learning_rate": 3.9952904238618524e-07,
      "logps/chosen": -235.05836486816406,
      "logps/rejected": -276.4933166503906,
      "loss": 0.1754,
      "rewards/chosen": -1.0856462717056274,
      "rewards/margins": 2.616638660430908,
      "rewards/rejected": -3.702285051345825,
      "step": 384
    },
    {
      "epoch": 0.2,
      "grad_norm": 9.603542307090086,
      "kl": 0.0,
      "learning_rate": 3.992673992673993e-07,
      "logps/chosen": -218.27806091308594,
      "logps/rejected": -227.58969116210938,
      "loss": 0.1948,
      "rewards/chosen": 0.6791697144508362,
      "rewards/margins": 3.0643129348754883,
      "rewards/rejected": -2.385143280029297,
      "step": 385
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.997346527322566,
      "kl": 0.0,
      "learning_rate": 3.9900575614861324e-07,
      "logps/chosen": -205.48524475097656,
      "logps/rejected": -237.6011505126953,
      "loss": 0.2379,
      "rewards/chosen": 0.24487002193927765,
      "rewards/margins": 2.6219630241394043,
      "rewards/rejected": -2.3770930767059326,
      "step": 386
    },
    {
      "epoch": 0.2,
      "grad_norm": 9.0666967002916,
      "kl": 0.0,
      "learning_rate": 3.987441130298273e-07,
      "logps/chosen": -196.8197479248047,
      "logps/rejected": -200.19735717773438,
      "loss": 0.1701,
      "rewards/chosen": 0.5998373627662659,
      "rewards/margins": 2.8779029846191406,
      "rewards/rejected": -2.2780656814575195,
      "step": 387
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.538513762529194,
      "kl": 0.0,
      "learning_rate": 3.9848246991104134e-07,
      "logps/chosen": -237.28199768066406,
      "logps/rejected": -266.018798828125,
      "loss": 0.167,
      "rewards/chosen": -0.623340904712677,
      "rewards/margins": 2.147143602371216,
      "rewards/rejected": -2.770484447479248,
      "step": 388
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.828887960825947,
      "kl": 0.0,
      "learning_rate": 3.9822082679225533e-07,
      "logps/chosen": -220.56329345703125,
      "logps/rejected": -268.723388671875,
      "loss": 0.1492,
      "rewards/chosen": 0.7558012008666992,
      "rewards/margins": 2.6395673751831055,
      "rewards/rejected": -1.8837661743164062,
      "step": 389
    },
    {
      "epoch": 0.2,
      "grad_norm": 11.47492309378771,
      "kl": 0.0,
      "learning_rate": 3.979591836734694e-07,
      "logps/chosen": -249.0001983642578,
      "logps/rejected": -240.189697265625,
      "loss": 0.18,
      "rewards/chosen": 0.21565745770931244,
      "rewards/margins": 2.2755565643310547,
      "rewards/rejected": -2.059899091720581,
      "step": 390
    },
    {
      "epoch": 0.2,
      "grad_norm": 8.926444305093995,
      "kl": 0.0,
      "learning_rate": 3.9769754055468343e-07,
      "logps/chosen": -200.0030059814453,
      "logps/rejected": -206.71624755859375,
      "loss": 0.1516,
      "rewards/chosen": -0.6393547058105469,
      "rewards/margins": 1.0063797235488892,
      "rewards/rejected": -1.645734429359436,
      "step": 391
    },
    {
      "epoch": 0.21,
      "grad_norm": 11.660568501998092,
      "kl": 0.0,
      "learning_rate": 3.974358974358974e-07,
      "logps/chosen": -223.32652282714844,
      "logps/rejected": -243.16845703125,
      "loss": 0.2899,
      "rewards/chosen": 0.2290610671043396,
      "rewards/margins": 1.6543824672698975,
      "rewards/rejected": -1.425321340560913,
      "step": 392
    },
    {
      "epoch": 0.21,
      "grad_norm": 14.42295762896043,
      "kl": 0.0,
      "learning_rate": 3.9717425431711143e-07,
      "logps/chosen": -289.037353515625,
      "logps/rejected": -252.30154418945312,
      "loss": 0.2241,
      "rewards/chosen": -1.7561075687408447,
      "rewards/margins": 0.43781447410583496,
      "rewards/rejected": -2.1939220428466797,
      "step": 393
    },
    {
      "epoch": 0.21,
      "grad_norm": 11.559459650147774,
      "kl": 0.0,
      "learning_rate": 3.969126111983255e-07,
      "logps/chosen": -245.2276153564453,
      "logps/rejected": -241.8197021484375,
      "loss": 0.2797,
      "rewards/chosen": -1.3280991315841675,
      "rewards/margins": 1.9113882780075073,
      "rewards/rejected": -3.239487409591675,
      "step": 394
    },
    {
      "epoch": 0.21,
      "grad_norm": 14.711575319555203,
      "kl": 0.0,
      "learning_rate": 3.9665096807953953e-07,
      "logps/chosen": -183.3538818359375,
      "logps/rejected": -245.3995819091797,
      "loss": 0.2133,
      "rewards/chosen": -0.0880654826760292,
      "rewards/margins": 2.2100796699523926,
      "rewards/rejected": -2.298145055770874,
      "step": 395
    },
    {
      "epoch": 0.21,
      "grad_norm": 13.58824047165676,
      "kl": 0.0,
      "learning_rate": 3.9638932496075353e-07,
      "logps/chosen": -319.79083251953125,
      "logps/rejected": -241.92750549316406,
      "loss": 0.2077,
      "rewards/chosen": -0.5814463496208191,
      "rewards/margins": 1.7764933109283447,
      "rewards/rejected": -2.3579397201538086,
      "step": 396
    },
    {
      "epoch": 0.21,
      "grad_norm": 10.51670285219746,
      "kl": 0.0,
      "learning_rate": 3.961276818419675e-07,
      "logps/chosen": -187.76580810546875,
      "logps/rejected": -264.5431823730469,
      "loss": 0.2047,
      "rewards/chosen": 0.11567319184541702,
      "rewards/margins": 4.132474899291992,
      "rewards/rejected": -4.016801834106445,
      "step": 397
    },
    {
      "epoch": 0.21,
      "grad_norm": 10.431439920064484,
      "kl": 0.0,
      "learning_rate": 3.958660387231816e-07,
      "logps/chosen": -152.51498413085938,
      "logps/rejected": -235.09324645996094,
      "loss": 0.2288,
      "rewards/chosen": 0.2288106083869934,
      "rewards/margins": 2.305171251296997,
      "rewards/rejected": -2.0763607025146484,
      "step": 398
    },
    {
      "epoch": 0.21,
      "grad_norm": 11.437728669932744,
      "kl": 0.0,
      "learning_rate": 3.9560439560439557e-07,
      "logps/chosen": -198.04653930664062,
      "logps/rejected": -216.70425415039062,
      "loss": 0.148,
      "rewards/chosen": 0.6912394165992737,
      "rewards/margins": 2.1111791133880615,
      "rewards/rejected": -1.4199397563934326,
      "step": 399
    },
    {
      "epoch": 0.21,
      "grad_norm": 10.47479277819163,
      "kl": 0.0,
      "learning_rate": 3.953427524856096e-07,
      "logps/chosen": -218.10205078125,
      "logps/rejected": -294.8763427734375,
      "loss": 0.1738,
      "rewards/chosen": 0.3303108215332031,
      "rewards/margins": 2.3888604640960693,
      "rewards/rejected": -2.058549642562866,
      "step": 400
    },
    {
      "epoch": 0.21,
      "grad_norm": 10.343750828726911,
      "kl": 0.0,
      "learning_rate": 3.9508110936682367e-07,
      "logps/chosen": -216.23837280273438,
      "logps/rejected": -222.16204833984375,
      "loss": 0.1711,
      "rewards/chosen": 0.24862945079803467,
      "rewards/margins": 2.214453935623169,
      "rewards/rejected": -1.9658244848251343,
      "step": 401
    },
    {
      "epoch": 0.21,
      "grad_norm": 11.335701218532215,
      "kl": 0.0,
      "learning_rate": 3.948194662480376e-07,
      "logps/chosen": -236.11170959472656,
      "logps/rejected": -206.62315368652344,
      "loss": 0.2024,
      "rewards/chosen": -0.06115901097655296,
      "rewards/margins": 1.5972472429275513,
      "rewards/rejected": -1.6584062576293945,
      "step": 402
    },
    {
      "epoch": 0.21,
      "grad_norm": 12.475003237928307,
      "kl": 0.0,
      "learning_rate": 3.9455782312925167e-07,
      "logps/chosen": -260.5793151855469,
      "logps/rejected": -287.6771545410156,
      "loss": 0.1706,
      "rewards/chosen": -0.39119893312454224,
      "rewards/margins": 1.3929164409637451,
      "rewards/rejected": -1.7841154336929321,
      "step": 403
    },
    {
      "epoch": 0.21,
      "grad_norm": 9.477036698967389,
      "kl": 0.0,
      "learning_rate": 3.942961800104657e-07,
      "logps/chosen": -175.79698181152344,
      "logps/rejected": -189.22921752929688,
      "loss": 0.1915,
      "rewards/chosen": 0.30524417757987976,
      "rewards/margins": 2.519529104232788,
      "rewards/rejected": -2.214284896850586,
      "step": 404
    },
    {
      "epoch": 0.21,
      "grad_norm": 12.239825540829113,
      "kl": 0.0,
      "learning_rate": 3.9403453689167977e-07,
      "logps/chosen": -271.8373718261719,
      "logps/rejected": -304.6640319824219,
      "loss": 0.1507,
      "rewards/chosen": -0.007900595664978027,
      "rewards/margins": 3.236818313598633,
      "rewards/rejected": -3.2447190284729004,
      "step": 405
    },
    {
      "epoch": 0.21,
      "grad_norm": 10.427157164376496,
      "kl": 0.0,
      "learning_rate": 3.9377289377289377e-07,
      "logps/chosen": -215.03746032714844,
      "logps/rejected": -255.59893798828125,
      "loss": 0.2518,
      "rewards/chosen": -0.6349587440490723,
      "rewards/margins": 2.439880847930908,
      "rewards/rejected": -3.0748395919799805,
      "step": 406
    },
    {
      "epoch": 0.21,
      "grad_norm": 11.207119574932996,
      "kl": 0.0,
      "learning_rate": 3.9351125065410776e-07,
      "logps/chosen": -167.9252471923828,
      "logps/rejected": -230.44461059570312,
      "loss": 0.2153,
      "rewards/chosen": -0.5827754139900208,
      "rewards/margins": 1.467712640762329,
      "rewards/rejected": -2.050487995147705,
      "step": 407
    },
    {
      "epoch": 0.21,
      "grad_norm": 12.843002882536329,
      "kl": 0.0,
      "learning_rate": 3.932496075353218e-07,
      "logps/chosen": -274.9442443847656,
      "logps/rejected": -291.43450927734375,
      "loss": 0.1899,
      "rewards/chosen": -0.4634897708892822,
      "rewards/margins": 2.9832985401153564,
      "rewards/rejected": -3.4467883110046387,
      "step": 408
    },
    {
      "epoch": 0.21,
      "grad_norm": 12.301578880284277,
      "kl": 0.0,
      "learning_rate": 3.929879644165358e-07,
      "logps/chosen": -230.11001586914062,
      "logps/rejected": -262.1216125488281,
      "loss": 0.2163,
      "rewards/chosen": -0.47498393058776855,
      "rewards/margins": 2.5229318141937256,
      "rewards/rejected": -2.997915744781494,
      "step": 409
    },
    {
      "epoch": 0.21,
      "grad_norm": 14.17354036141092,
      "kl": 0.0,
      "learning_rate": 3.9272632129774986e-07,
      "logps/chosen": -236.7783203125,
      "logps/rejected": -225.37086486816406,
      "loss": 0.1465,
      "rewards/chosen": 0.534197211265564,
      "rewards/margins": 3.3101611137390137,
      "rewards/rejected": -2.7759640216827393,
      "step": 410
    },
    {
      "epoch": 0.22,
      "grad_norm": 9.994681734611301,
      "kl": 0.0,
      "learning_rate": 3.924646781789639e-07,
      "logps/chosen": -152.39027404785156,
      "logps/rejected": -236.91995239257812,
      "loss": 0.1985,
      "rewards/chosen": -0.45504745841026306,
      "rewards/margins": 2.5142714977264404,
      "rewards/rejected": -2.9693188667297363,
      "step": 411
    },
    {
      "epoch": 0.22,
      "grad_norm": 9.62216210519223,
      "kl": 0.0,
      "learning_rate": 3.9220303506017786e-07,
      "logps/chosen": -222.86874389648438,
      "logps/rejected": -318.4061279296875,
      "loss": 0.1912,
      "rewards/chosen": -0.2937797009944916,
      "rewards/margins": 3.895700216293335,
      "rewards/rejected": -4.189479827880859,
      "step": 412
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.901795525608296,
      "kl": 0.0,
      "learning_rate": 3.919413919413919e-07,
      "logps/chosen": -214.8961639404297,
      "logps/rejected": -260.2914733886719,
      "loss": 0.2245,
      "rewards/chosen": 0.16179826855659485,
      "rewards/margins": 2.0142719745635986,
      "rewards/rejected": -1.8524737358093262,
      "step": 413
    },
    {
      "epoch": 0.22,
      "grad_norm": 13.652487549232319,
      "kl": 0.0,
      "learning_rate": 3.9167974882260596e-07,
      "logps/chosen": -212.1907958984375,
      "logps/rejected": -278.97296142578125,
      "loss": 0.1698,
      "rewards/chosen": -0.08914380520582199,
      "rewards/margins": 3.5527141094207764,
      "rewards/rejected": -3.641857862472534,
      "step": 414
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.750735909217193,
      "kl": 0.0,
      "learning_rate": 3.9141810570381995e-07,
      "logps/chosen": -219.64820861816406,
      "logps/rejected": -223.59970092773438,
      "loss": 0.1419,
      "rewards/chosen": -0.31567585468292236,
      "rewards/margins": 1.8360644578933716,
      "rewards/rejected": -2.151740312576294,
      "step": 415
    },
    {
      "epoch": 0.22,
      "grad_norm": 13.00956558924868,
      "kl": 0.0,
      "learning_rate": 3.91156462585034e-07,
      "logps/chosen": -221.29444885253906,
      "logps/rejected": -292.55242919921875,
      "loss": 0.2404,
      "rewards/chosen": -0.8755928874015808,
      "rewards/margins": 2.879188060760498,
      "rewards/rejected": -3.7547810077667236,
      "step": 416
    },
    {
      "epoch": 0.22,
      "grad_norm": 11.263246462543782,
      "kl": 0.0,
      "learning_rate": 3.90894819466248e-07,
      "logps/chosen": -204.36553955078125,
      "logps/rejected": -277.9330749511719,
      "loss": 0.2751,
      "rewards/chosen": -0.16093337535858154,
      "rewards/margins": 1.7333406209945679,
      "rewards/rejected": -1.8942739963531494,
      "step": 417
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.77749694683554,
      "kl": 0.0,
      "learning_rate": 3.9063317634746205e-07,
      "logps/chosen": -218.55001831054688,
      "logps/rejected": -276.6634216308594,
      "loss": 0.1298,
      "rewards/chosen": 0.9015990495681763,
      "rewards/margins": 3.651735305786133,
      "rewards/rejected": -2.750136137008667,
      "step": 418
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.516288450325918,
      "kl": 0.0,
      "learning_rate": 3.9037153322867605e-07,
      "logps/chosen": -210.36167907714844,
      "logps/rejected": -265.4808654785156,
      "loss": 0.2017,
      "rewards/chosen": -0.17863792181015015,
      "rewards/margins": 2.6375937461853027,
      "rewards/rejected": -2.8162317276000977,
      "step": 419
    },
    {
      "epoch": 0.22,
      "grad_norm": 11.824811555332477,
      "kl": 0.0,
      "learning_rate": 3.901098901098901e-07,
      "logps/chosen": -208.4647674560547,
      "logps/rejected": -255.76190185546875,
      "loss": 0.199,
      "rewards/chosen": 0.15355457365512848,
      "rewards/margins": 2.69331955909729,
      "rewards/rejected": -2.539764881134033,
      "step": 420
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.69862877229561,
      "kl": 0.0,
      "learning_rate": 3.8984824699110415e-07,
      "logps/chosen": -256.15496826171875,
      "logps/rejected": -272.2501220703125,
      "loss": 0.1392,
      "rewards/chosen": 0.36112552881240845,
      "rewards/margins": 3.3303704261779785,
      "rewards/rejected": -2.969244956970215,
      "step": 421
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.317432492795456,
      "kl": 0.0,
      "learning_rate": 3.895866038723181e-07,
      "logps/chosen": -201.2956085205078,
      "logps/rejected": -252.26727294921875,
      "loss": 0.2015,
      "rewards/chosen": -0.19176657497882843,
      "rewards/margins": 1.5057237148284912,
      "rewards/rejected": -1.6974903345108032,
      "step": 422
    },
    {
      "epoch": 0.22,
      "grad_norm": 11.089829796355776,
      "kl": 0.0,
      "learning_rate": 3.8932496075353214e-07,
      "logps/chosen": -251.06739807128906,
      "logps/rejected": -207.30560302734375,
      "loss": 0.2024,
      "rewards/chosen": 0.43589267134666443,
      "rewards/margins": 2.9610509872436523,
      "rewards/rejected": -2.525158405303955,
      "step": 423
    },
    {
      "epoch": 0.22,
      "grad_norm": 13.046985449887648,
      "kl": 0.0,
      "learning_rate": 3.890633176347462e-07,
      "logps/chosen": -191.86936950683594,
      "logps/rejected": -282.1868896484375,
      "loss": 0.172,
      "rewards/chosen": -0.3784329891204834,
      "rewards/margins": 2.067564010620117,
      "rewards/rejected": -2.4459969997406006,
      "step": 424
    },
    {
      "epoch": 0.22,
      "grad_norm": 9.765346020761122,
      "kl": 0.0,
      "learning_rate": 3.888016745159602e-07,
      "logps/chosen": -199.694580078125,
      "logps/rejected": -256.0482482910156,
      "loss": 0.1586,
      "rewards/chosen": -0.6785757541656494,
      "rewards/margins": 1.9910736083984375,
      "rewards/rejected": -2.669649362564087,
      "step": 425
    },
    {
      "epoch": 0.22,
      "grad_norm": 14.196677493751132,
      "kl": 0.0,
      "learning_rate": 3.8854003139717424e-07,
      "logps/chosen": -228.76034545898438,
      "logps/rejected": -302.63836669921875,
      "loss": 0.1734,
      "rewards/chosen": -0.3336869776248932,
      "rewards/margins": 2.1534783840179443,
      "rewards/rejected": -2.4871654510498047,
      "step": 426
    },
    {
      "epoch": 0.22,
      "grad_norm": 11.772671674738493,
      "kl": 0.0,
      "learning_rate": 3.882783882783883e-07,
      "logps/chosen": -234.97866821289062,
      "logps/rejected": -277.7060241699219,
      "loss": 0.2091,
      "rewards/chosen": -0.5110215544700623,
      "rewards/margins": 0.7004496455192566,
      "rewards/rejected": -1.2114711999893188,
      "step": 427
    },
    {
      "epoch": 0.22,
      "grad_norm": 13.524888701607797,
      "kl": 0.0,
      "learning_rate": 3.880167451596023e-07,
      "logps/chosen": -222.4050750732422,
      "logps/rejected": -227.77481079101562,
      "loss": 0.2568,
      "rewards/chosen": 0.0008676449651829898,
      "rewards/margins": 1.8511059284210205,
      "rewards/rejected": -1.85023832321167,
      "step": 428
    },
    {
      "epoch": 0.22,
      "grad_norm": 10.405779261311427,
      "kl": 0.0,
      "learning_rate": 3.877551020408163e-07,
      "logps/chosen": -204.4320068359375,
      "logps/rejected": -222.79629516601562,
      "loss": 0.0875,
      "rewards/chosen": 0.08031473308801651,
      "rewards/margins": 2.481466770172119,
      "rewards/rejected": -2.4011521339416504,
      "step": 429
    },
    {
      "epoch": 0.23,
      "grad_norm": 14.72924638814786,
      "kl": 0.0,
      "learning_rate": 3.8749345892203034e-07,
      "logps/chosen": -289.6814270019531,
      "logps/rejected": -331.42816162109375,
      "loss": 0.1481,
      "rewards/chosen": -0.13640549778938293,
      "rewards/margins": 4.516214370727539,
      "rewards/rejected": -4.6526198387146,
      "step": 430
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.741301498332556,
      "kl": 0.0,
      "learning_rate": 3.872318158032444e-07,
      "logps/chosen": -222.26815795898438,
      "logps/rejected": -271.1949462890625,
      "loss": 0.197,
      "rewards/chosen": 0.39989006519317627,
      "rewards/margins": 2.3632915019989014,
      "rewards/rejected": -1.963401436805725,
      "step": 431
    },
    {
      "epoch": 0.23,
      "grad_norm": 12.112546116412723,
      "kl": 0.0,
      "learning_rate": 3.869701726844584e-07,
      "logps/chosen": -243.7536163330078,
      "logps/rejected": -252.95794677734375,
      "loss": 0.2012,
      "rewards/chosen": -0.38057583570480347,
      "rewards/margins": 1.647994041442871,
      "rewards/rejected": -2.0285699367523193,
      "step": 432
    },
    {
      "epoch": 0.23,
      "grad_norm": 14.742669713065881,
      "kl": 0.0,
      "learning_rate": 3.867085295656724e-07,
      "logps/chosen": -244.42164611816406,
      "logps/rejected": -267.96954345703125,
      "loss": 0.2001,
      "rewards/chosen": 0.29253336787223816,
      "rewards/margins": 2.2655460834503174,
      "rewards/rejected": -1.9730128049850464,
      "step": 433
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.134298526921,
      "kl": 0.0,
      "learning_rate": 3.8644688644688643e-07,
      "logps/chosen": -193.90284729003906,
      "logps/rejected": -232.6556396484375,
      "loss": 0.2008,
      "rewards/chosen": 0.34420499205589294,
      "rewards/margins": 3.2133047580718994,
      "rewards/rejected": -2.8690998554229736,
      "step": 434
    },
    {
      "epoch": 0.23,
      "grad_norm": 11.746178628630169,
      "kl": 0.0,
      "learning_rate": 3.8618524332810043e-07,
      "logps/chosen": -220.666015625,
      "logps/rejected": -303.94451904296875,
      "loss": 0.1645,
      "rewards/chosen": 0.019716663286089897,
      "rewards/margins": 2.469646692276001,
      "rewards/rejected": -2.44992995262146,
      "step": 435
    },
    {
      "epoch": 0.23,
      "grad_norm": 11.618541405718464,
      "kl": 0.0,
      "learning_rate": 3.859236002093145e-07,
      "logps/chosen": -201.02305603027344,
      "logps/rejected": -307.0491027832031,
      "loss": 0.1626,
      "rewards/chosen": -0.7490205764770508,
      "rewards/margins": 2.6788406372070312,
      "rewards/rejected": -3.427861213684082,
      "step": 436
    },
    {
      "epoch": 0.23,
      "grad_norm": 12.168910387907376,
      "kl": 0.0,
      "learning_rate": 3.8566195709052853e-07,
      "logps/chosen": -278.5819396972656,
      "logps/rejected": -261.8109130859375,
      "loss": 0.0773,
      "rewards/chosen": -0.5979146361351013,
      "rewards/margins": 2.49701189994812,
      "rewards/rejected": -3.094926595687866,
      "step": 437
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.604103054919578,
      "kl": 0.0,
      "learning_rate": 3.8540031397174253e-07,
      "logps/chosen": -193.6268768310547,
      "logps/rejected": -241.7948455810547,
      "loss": 0.1785,
      "rewards/chosen": 0.43937230110168457,
      "rewards/margins": 1.834383487701416,
      "rewards/rejected": -1.3950111865997314,
      "step": 438
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.834100142745722,
      "kl": 0.0,
      "learning_rate": 3.851386708529565e-07,
      "logps/chosen": -229.8187713623047,
      "logps/rejected": -325.75762939453125,
      "loss": 0.1844,
      "rewards/chosen": 0.6806701421737671,
      "rewards/margins": 4.323229789733887,
      "rewards/rejected": -3.64255952835083,
      "step": 439
    },
    {
      "epoch": 0.23,
      "grad_norm": 12.704909503144034,
      "kl": 0.0,
      "learning_rate": 3.848770277341706e-07,
      "logps/chosen": -224.91845703125,
      "logps/rejected": -340.0356750488281,
      "loss": 0.2631,
      "rewards/chosen": 0.026007119566202164,
      "rewards/margins": 2.283827781677246,
      "rewards/rejected": -2.2578206062316895,
      "step": 440
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.41021619314634,
      "kl": 0.0,
      "learning_rate": 3.8461538461538463e-07,
      "logps/chosen": -219.94329833984375,
      "logps/rejected": -233.54290771484375,
      "loss": 0.1716,
      "rewards/chosen": -0.22441209852695465,
      "rewards/margins": 2.2788405418395996,
      "rewards/rejected": -2.5032527446746826,
      "step": 441
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.379317080534767,
      "kl": 0.0,
      "learning_rate": 3.843537414965986e-07,
      "logps/chosen": -229.6413116455078,
      "logps/rejected": -249.2711639404297,
      "loss": 0.1614,
      "rewards/chosen": 0.3848222494125366,
      "rewards/margins": 1.9725074768066406,
      "rewards/rejected": -1.587685227394104,
      "step": 442
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.520229093007123,
      "kl": 0.0,
      "learning_rate": 3.840920983778126e-07,
      "logps/chosen": -222.9331817626953,
      "logps/rejected": -208.89297485351562,
      "loss": 0.1427,
      "rewards/chosen": 0.582370400428772,
      "rewards/margins": 3.301401138305664,
      "rewards/rejected": -2.7190306186676025,
      "step": 443
    },
    {
      "epoch": 0.23,
      "grad_norm": 13.275687424817486,
      "kl": 0.0,
      "learning_rate": 3.8383045525902667e-07,
      "logps/chosen": -189.17926025390625,
      "logps/rejected": -219.13580322265625,
      "loss": 0.2513,
      "rewards/chosen": -0.015819329768419266,
      "rewards/margins": 1.8377530574798584,
      "rewards/rejected": -1.8535723686218262,
      "step": 444
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.600324116674757,
      "kl": 0.0,
      "learning_rate": 3.8356881214024067e-07,
      "logps/chosen": -216.97711181640625,
      "logps/rejected": -249.7940216064453,
      "loss": 0.2049,
      "rewards/chosen": 0.5193243622779846,
      "rewards/margins": 2.258141040802002,
      "rewards/rejected": -1.738816738128662,
      "step": 445
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.525139117282963,
      "kl": 0.0,
      "learning_rate": 3.833071690214547e-07,
      "logps/chosen": -222.4760284423828,
      "logps/rejected": -239.29403686523438,
      "loss": 0.071,
      "rewards/chosen": 0.5078960657119751,
      "rewards/margins": 2.6218113899230957,
      "rewards/rejected": -2.113915205001831,
      "step": 446
    },
    {
      "epoch": 0.23,
      "grad_norm": 13.222155048640982,
      "kl": 0.0,
      "learning_rate": 3.8304552590266877e-07,
      "logps/chosen": -200.71603393554688,
      "logps/rejected": -267.3768005371094,
      "loss": 0.162,
      "rewards/chosen": 0.24389874935150146,
      "rewards/margins": 3.596365451812744,
      "rewards/rejected": -3.3524668216705322,
      "step": 447
    },
    {
      "epoch": 0.23,
      "grad_norm": 10.120823838278643,
      "kl": 0.0,
      "learning_rate": 3.827838827838827e-07,
      "logps/chosen": -196.2765350341797,
      "logps/rejected": -215.92112731933594,
      "loss": 0.1333,
      "rewards/chosen": 0.05430547147989273,
      "rewards/margins": 1.7621450424194336,
      "rewards/rejected": -1.7078396081924438,
      "step": 448
    },
    {
      "epoch": 0.23,
      "grad_norm": 11.458337974039024,
      "kl": 0.0,
      "learning_rate": 3.8252223966509676e-07,
      "logps/chosen": -195.0396270751953,
      "logps/rejected": -285.74462890625,
      "loss": 0.1939,
      "rewards/chosen": 0.07916612178087234,
      "rewards/margins": 2.918091058731079,
      "rewards/rejected": -2.8389248847961426,
      "step": 449
    },
    {
      "epoch": 0.24,
      "grad_norm": 12.160182938253959,
      "kl": 0.0,
      "learning_rate": 3.822605965463108e-07,
      "logps/chosen": -193.6781768798828,
      "logps/rejected": -243.92469787597656,
      "loss": 0.1757,
      "rewards/chosen": -0.5648252964019775,
      "rewards/margins": 1.6671717166900635,
      "rewards/rejected": -2.231997013092041,
      "step": 450
    },
    {
      "epoch": 0.24,
      "grad_norm": 12.143861035511275,
      "kl": 0.0,
      "learning_rate": 3.8199895342752486e-07,
      "logps/chosen": -208.96681213378906,
      "logps/rejected": -243.71438598632812,
      "loss": 0.1589,
      "rewards/chosen": -0.6121307015419006,
      "rewards/margins": 1.7129931449890137,
      "rewards/rejected": -2.3251237869262695,
      "step": 451
    },
    {
      "epoch": 0.24,
      "grad_norm": 9.91525808504644,
      "kl": 0.0,
      "learning_rate": 3.8173731030873886e-07,
      "logps/chosen": -266.3404541015625,
      "logps/rejected": -302.9526672363281,
      "loss": 0.1843,
      "rewards/chosen": -0.14618553221225739,
      "rewards/margins": 2.309356689453125,
      "rewards/rejected": -2.4555423259735107,
      "step": 452
    },
    {
      "epoch": 0.24,
      "grad_norm": 9.702336587181945,
      "kl": 0.0,
      "learning_rate": 3.814756671899529e-07,
      "logps/chosen": -181.5204620361328,
      "logps/rejected": -304.9017639160156,
      "loss": 0.1483,
      "rewards/chosen": -0.1943751722574234,
      "rewards/margins": 3.55947208404541,
      "rewards/rejected": -3.753847360610962,
      "step": 453
    },
    {
      "epoch": 0.24,
      "grad_norm": 11.965604322215004,
      "kl": 0.0,
      "learning_rate": 3.812140240711669e-07,
      "logps/chosen": -245.6624755859375,
      "logps/rejected": -260.020263671875,
      "loss": 0.2183,
      "rewards/chosen": -1.2427337169647217,
      "rewards/margins": 1.6019926071166992,
      "rewards/rejected": -2.844726324081421,
      "step": 454
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.836340963305517,
      "kl": 0.0,
      "learning_rate": 3.809523809523809e-07,
      "logps/chosen": -237.24041748046875,
      "logps/rejected": -235.87216186523438,
      "loss": 0.1095,
      "rewards/chosen": 0.8246784210205078,
      "rewards/margins": 3.6156461238861084,
      "rewards/rejected": -2.7909677028656006,
      "step": 455
    },
    {
      "epoch": 0.24,
      "grad_norm": 11.897690499315637,
      "kl": 0.0,
      "learning_rate": 3.8069073783359496e-07,
      "logps/chosen": -232.65216064453125,
      "logps/rejected": -197.15951538085938,
      "loss": 0.0782,
      "rewards/chosen": 0.05020507052540779,
      "rewards/margins": 3.441476821899414,
      "rewards/rejected": -3.3912718296051025,
      "step": 456
    },
    {
      "epoch": 0.24,
      "grad_norm": 8.049103139267938,
      "kl": 0.0,
      "learning_rate": 3.80429094714809e-07,
      "logps/chosen": -239.66998291015625,
      "logps/rejected": -256.76837158203125,
      "loss": 0.1933,
      "rewards/chosen": 0.054171182215213776,
      "rewards/margins": 3.8353452682495117,
      "rewards/rejected": -3.7811741828918457,
      "step": 457
    },
    {
      "epoch": 0.24,
      "grad_norm": 13.088097317739122,
      "kl": 0.0,
      "learning_rate": 3.80167451596023e-07,
      "logps/chosen": -205.0490264892578,
      "logps/rejected": -241.65966796875,
      "loss": 0.1946,
      "rewards/chosen": -0.03932800143957138,
      "rewards/margins": 2.7577974796295166,
      "rewards/rejected": -2.7971255779266357,
      "step": 458
    },
    {
      "epoch": 0.24,
      "grad_norm": 13.14569665419969,
      "kl": 0.0,
      "learning_rate": 3.79905808477237e-07,
      "logps/chosen": -298.6637268066406,
      "logps/rejected": -285.0900573730469,
      "loss": 0.1677,
      "rewards/chosen": -0.5165401697158813,
      "rewards/margins": 3.1756558418273926,
      "rewards/rejected": -3.6921958923339844,
      "step": 459
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.434555514022984,
      "kl": 0.0,
      "learning_rate": 3.7964416535845105e-07,
      "logps/chosen": -228.3349609375,
      "logps/rejected": -214.828857421875,
      "loss": 0.1609,
      "rewards/chosen": 0.11236274987459183,
      "rewards/margins": 2.25835919380188,
      "rewards/rejected": -2.145996332168579,
      "step": 460
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.167277710211138,
      "kl": 0.0,
      "learning_rate": 3.793825222396651e-07,
      "logps/chosen": -245.6664581298828,
      "logps/rejected": -274.6752624511719,
      "loss": 0.1496,
      "rewards/chosen": -0.27803435921669006,
      "rewards/margins": 1.9288045167922974,
      "rewards/rejected": -2.206838846206665,
      "step": 461
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.474016211401475,
      "kl": 0.0,
      "learning_rate": 3.791208791208791e-07,
      "logps/chosen": -193.68919372558594,
      "logps/rejected": -202.5056610107422,
      "loss": 0.1846,
      "rewards/chosen": -0.4363885223865509,
      "rewards/margins": 2.7338576316833496,
      "rewards/rejected": -3.170246124267578,
      "step": 462
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.659154195676038,
      "kl": 0.0,
      "learning_rate": 3.7885923600209315e-07,
      "logps/chosen": -242.42816162109375,
      "logps/rejected": -282.0824890136719,
      "loss": 0.1603,
      "rewards/chosen": 0.673112690448761,
      "rewards/margins": 3.3613035678863525,
      "rewards/rejected": -2.6881909370422363,
      "step": 463
    },
    {
      "epoch": 0.24,
      "grad_norm": 12.65492990351007,
      "kl": 0.0,
      "learning_rate": 3.7859759288330715e-07,
      "logps/chosen": -170.49734497070312,
      "logps/rejected": -210.32472229003906,
      "loss": 0.1996,
      "rewards/chosen": 0.2279433161020279,
      "rewards/margins": 2.666241407394409,
      "rewards/rejected": -2.438297986984253,
      "step": 464
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.54524759203527,
      "kl": 0.0,
      "learning_rate": 3.7833594976452115e-07,
      "logps/chosen": -153.5712432861328,
      "logps/rejected": -269.2149353027344,
      "loss": 0.2098,
      "rewards/chosen": -0.019966602325439453,
      "rewards/margins": 2.5769615173339844,
      "rewards/rejected": -2.596928119659424,
      "step": 465
    },
    {
      "epoch": 0.24,
      "grad_norm": 10.51825030268249,
      "kl": 0.0,
      "learning_rate": 3.780743066457352e-07,
      "logps/chosen": -203.99156188964844,
      "logps/rejected": -266.8109436035156,
      "loss": 0.1801,
      "rewards/chosen": 1.9818308353424072,
      "rewards/margins": 3.033543109893799,
      "rewards/rejected": -1.051712155342102,
      "step": 466
    },
    {
      "epoch": 0.24,
      "grad_norm": 9.77436331647093,
      "kl": 0.0,
      "learning_rate": 3.7781266352694925e-07,
      "logps/chosen": -240.96786499023438,
      "logps/rejected": -226.21963500976562,
      "loss": 0.146,
      "rewards/chosen": -0.04570434242486954,
      "rewards/margins": 1.6731890439987183,
      "rewards/rejected": -1.7188934087753296,
      "step": 467
    },
    {
      "epoch": 0.24,
      "grad_norm": 11.33142669534873,
      "kl": 0.0,
      "learning_rate": 3.7755102040816324e-07,
      "logps/chosen": -274.2488708496094,
      "logps/rejected": -221.2158203125,
      "loss": 0.2061,
      "rewards/chosen": -0.03258565813302994,
      "rewards/margins": 1.8558272123336792,
      "rewards/rejected": -1.8884128332138062,
      "step": 468
    },
    {
      "epoch": 0.25,
      "grad_norm": 12.417447396754456,
      "kl": 0.0,
      "learning_rate": 3.7728937728937724e-07,
      "logps/chosen": -241.2676544189453,
      "logps/rejected": -312.99432373046875,
      "loss": 0.1357,
      "rewards/chosen": 1.6663408279418945,
      "rewards/margins": 3.927600383758545,
      "rewards/rejected": -2.2612595558166504,
      "step": 469
    },
    {
      "epoch": 0.25,
      "grad_norm": 11.170512277856618,
      "kl": 0.0,
      "learning_rate": 3.770277341705913e-07,
      "logps/chosen": -233.8650360107422,
      "logps/rejected": -197.24072265625,
      "loss": 0.1556,
      "rewards/chosen": 0.8342301249504089,
      "rewards/margins": 2.448633909225464,
      "rewards/rejected": -1.6144037246704102,
      "step": 470
    },
    {
      "epoch": 0.25,
      "grad_norm": 13.803253821404509,
      "kl": 0.0,
      "learning_rate": 3.7676609105180534e-07,
      "logps/chosen": -194.3175506591797,
      "logps/rejected": -353.5451354980469,
      "loss": 0.1461,
      "rewards/chosen": 0.6489688754081726,
      "rewards/margins": 3.0008530616760254,
      "rewards/rejected": -2.351884126663208,
      "step": 471
    },
    {
      "epoch": 0.25,
      "grad_norm": 11.510104873936742,
      "kl": 0.0,
      "learning_rate": 3.7650444793301934e-07,
      "logps/chosen": -276.4261474609375,
      "logps/rejected": -173.75657653808594,
      "loss": 0.2384,
      "rewards/chosen": -0.2080867886543274,
      "rewards/margins": 0.7136066555976868,
      "rewards/rejected": -0.9216934442520142,
      "step": 472
    },
    {
      "epoch": 0.25,
      "grad_norm": 11.151674395455558,
      "kl": 0.0,
      "learning_rate": 3.762428048142334e-07,
      "logps/chosen": -234.051025390625,
      "logps/rejected": -349.57586669921875,
      "loss": 0.1555,
      "rewards/chosen": 0.1754276305437088,
      "rewards/margins": 2.250746965408325,
      "rewards/rejected": -2.075319290161133,
      "step": 473
    },
    {
      "epoch": 0.25,
      "grad_norm": 10.012473838900807,
      "kl": 0.0,
      "learning_rate": 3.759811616954474e-07,
      "logps/chosen": -206.8048553466797,
      "logps/rejected": -245.5289306640625,
      "loss": 0.1644,
      "rewards/chosen": 0.046800464391708374,
      "rewards/margins": 2.0636954307556152,
      "rewards/rejected": -2.016895055770874,
      "step": 474
    },
    {
      "epoch": 0.25,
      "grad_norm": 12.41419451120897,
      "kl": 0.0,
      "learning_rate": 3.757195185766614e-07,
      "logps/chosen": -197.76080322265625,
      "logps/rejected": -243.91220092773438,
      "loss": 0.1441,
      "rewards/chosen": 0.16484998166561127,
      "rewards/margins": 1.870893120765686,
      "rewards/rejected": -1.7060431241989136,
      "step": 475
    },
    {
      "epoch": 0.25,
      "grad_norm": 12.281595873200606,
      "kl": 0.0,
      "learning_rate": 3.7545787545787543e-07,
      "logps/chosen": -206.76541137695312,
      "logps/rejected": -230.58517456054688,
      "loss": 0.1835,
      "rewards/chosen": 0.500891923904419,
      "rewards/margins": 2.8288941383361816,
      "rewards/rejected": -2.3280022144317627,
      "step": 476
    },
    {
      "epoch": 0.25,
      "grad_norm": 11.477594673948579,
      "kl": 0.0,
      "learning_rate": 3.751962323390895e-07,
      "logps/chosen": -205.23182678222656,
      "logps/rejected": -265.99298095703125,
      "loss": 0.2508,
      "rewards/chosen": -0.7120283842086792,
      "rewards/margins": 1.5744119882583618,
      "rewards/rejected": -2.286440372467041,
      "step": 477
    },
    {
      "epoch": 0.25,
      "grad_norm": 10.389487469578903,
      "kl": 0.0,
      "learning_rate": 3.749345892203035e-07,
      "logps/chosen": -239.08309936523438,
      "logps/rejected": -245.90695190429688,
      "loss": 0.2403,
      "rewards/chosen": 0.03259509429335594,
      "rewards/margins": 1.3131746053695679,
      "rewards/rejected": -1.2805795669555664,
      "step": 478
    },
    {
      "epoch": 0.25,
      "grad_norm": 12.417872067981774,
      "kl": 0.0,
      "learning_rate": 3.7467294610151753e-07,
      "logps/chosen": -256.7392883300781,
      "logps/rejected": -276.9643249511719,
      "loss": 0.2144,
      "rewards/chosen": 0.17145968973636627,
      "rewards/margins": 2.2896573543548584,
      "rewards/rejected": -2.1181976795196533,
      "step": 479
    },
    {
      "epoch": 0.25,
      "grad_norm": 11.327729837593566,
      "kl": 0.0,
      "learning_rate": 3.7441130298273153e-07,
      "logps/chosen": -194.67178344726562,
      "logps/rejected": -293.6398620605469,
      "loss": 0.2087,
      "rewards/chosen": 0.07769111543893814,
      "rewards/margins": 2.251216411590576,
      "rewards/rejected": -2.173525333404541,
      "step": 480
    },
    {
      "epoch": 0.25,
      "grad_norm": 11.641851328355145,
      "kl": 0.0,
      "learning_rate": 3.741496598639456e-07,
      "logps/chosen": -225.63128662109375,
      "logps/rejected": -242.99880981445312,
      "loss": 0.2983,
      "rewards/chosen": -0.29468005895614624,
      "rewards/margins": 1.5418860912322998,
      "rewards/rejected": -1.8365662097930908,
      "step": 481
    },
    {
      "epoch": 0.25,
      "grad_norm": 10.688981054253707,
      "kl": 0.0,
      "learning_rate": 3.738880167451596e-07,
      "logps/chosen": -194.8035125732422,
      "logps/rejected": -327.953369140625,
      "loss": 0.1177,
      "rewards/chosen": 0.040647171437740326,
      "rewards/margins": 2.5855929851531982,
      "rewards/rejected": -2.54494571685791,
      "step": 482
    },
    {
      "epoch": 0.25,
      "grad_norm": 10.545945342907165,
      "kl": 0.0,
      "learning_rate": 3.7362637362637363e-07,
      "logps/chosen": -299.9187927246094,
      "logps/rejected": -232.39605712890625,
      "loss": 0.142,
      "rewards/chosen": 0.8184850811958313,
      "rewards/margins": 3.6689701080322266,
      "rewards/rejected": -2.85048508644104,
      "step": 483
    },
    {
      "epoch": 0.25,
      "grad_norm": 12.552799610744065,
      "kl": 0.0,
      "learning_rate": 3.733647305075877e-07,
      "logps/chosen": -239.55209350585938,
      "logps/rejected": -253.3202667236328,
      "loss": 0.1485,
      "rewards/chosen": -0.060923367738723755,
      "rewards/margins": 1.103141188621521,
      "rewards/rejected": -1.1640645265579224,
      "step": 484
    },
    {
      "epoch": 0.25,
      "grad_norm": 10.114942413331908,
      "kl": 0.0,
      "learning_rate": 3.731030873888016e-07,
      "logps/chosen": -188.8922882080078,
      "logps/rejected": -213.3092041015625,
      "loss": 0.1095,
      "rewards/chosen": 0.31580618023872375,
      "rewards/margins": 2.5469398498535156,
      "rewards/rejected": -2.2311336994171143,
      "step": 485
    },
    {
      "epoch": 0.25,
      "grad_norm": 14.076487746859804,
      "kl": 0.0,
      "learning_rate": 3.7284144427001567e-07,
      "logps/chosen": -252.74615478515625,
      "logps/rejected": -206.2298583984375,
      "loss": 0.1541,
      "rewards/chosen": 0.32222241163253784,
      "rewards/margins": 2.381617307662964,
      "rewards/rejected": -2.0593948364257812,
      "step": 486
    },
    {
      "epoch": 0.25,
      "grad_norm": 13.614522220242236,
      "kl": 0.0,
      "learning_rate": 3.725798011512297e-07,
      "logps/chosen": -233.2279052734375,
      "logps/rejected": -309.02667236328125,
      "loss": 0.1624,
      "rewards/chosen": 0.3280530273914337,
      "rewards/margins": 2.868494987487793,
      "rewards/rejected": -2.5404419898986816,
      "step": 487
    },
    {
      "epoch": 0.26,
      "grad_norm": 13.04079850351043,
      "kl": 0.0,
      "learning_rate": 3.723181580324437e-07,
      "logps/chosen": -236.299560546875,
      "logps/rejected": -235.88479614257812,
      "loss": 0.2823,
      "rewards/chosen": -0.19492840766906738,
      "rewards/margins": 2.121401786804199,
      "rewards/rejected": -2.3163301944732666,
      "step": 488
    },
    {
      "epoch": 0.26,
      "grad_norm": 13.854146801900269,
      "kl": 0.0,
      "learning_rate": 3.7205651491365777e-07,
      "logps/chosen": -220.61044311523438,
      "logps/rejected": -256.89471435546875,
      "loss": 0.2039,
      "rewards/chosen": 0.09335081279277802,
      "rewards/margins": 1.3945070505142212,
      "rewards/rejected": -1.3011562824249268,
      "step": 489
    },
    {
      "epoch": 0.26,
      "grad_norm": 12.146701230600012,
      "kl": 0.0,
      "learning_rate": 3.7179487179487177e-07,
      "logps/chosen": -244.57260131835938,
      "logps/rejected": -216.97161865234375,
      "loss": 0.1947,
      "rewards/chosen": 0.42316800355911255,
      "rewards/margins": 1.4870796203613281,
      "rewards/rejected": -1.0639115571975708,
      "step": 490
    },
    {
      "epoch": 0.26,
      "grad_norm": 10.730648049345493,
      "kl": 0.0,
      "learning_rate": 3.7153322867608577e-07,
      "logps/chosen": -218.7908935546875,
      "logps/rejected": -258.998291015625,
      "loss": 0.183,
      "rewards/chosen": -0.3176134526729584,
      "rewards/margins": 1.3556625843048096,
      "rewards/rejected": -1.6732760667800903,
      "step": 491
    },
    {
      "epoch": 0.26,
      "grad_norm": 10.23814496690706,
      "kl": 0.0,
      "learning_rate": 3.712715855572998e-07,
      "logps/chosen": -223.61936950683594,
      "logps/rejected": -253.8956756591797,
      "loss": 0.1727,
      "rewards/chosen": 0.5035489201545715,
      "rewards/margins": 2.772200107574463,
      "rewards/rejected": -2.268651247024536,
      "step": 492
    },
    {
      "epoch": 0.26,
      "grad_norm": 12.792135627932897,
      "kl": 0.0,
      "learning_rate": 3.7100994243851387e-07,
      "logps/chosen": -254.05380249023438,
      "logps/rejected": -232.8262939453125,
      "loss": 0.1926,
      "rewards/chosen": 1.061255693435669,
      "rewards/margins": 2.6245744228363037,
      "rewards/rejected": -1.5633187294006348,
      "step": 493
    },
    {
      "epoch": 0.26,
      "grad_norm": 11.345332305270757,
      "kl": 0.0,
      "learning_rate": 3.707482993197279e-07,
      "logps/chosen": -232.64930725097656,
      "logps/rejected": -215.4774627685547,
      "loss": 0.1479,
      "rewards/chosen": 0.9204630851745605,
      "rewards/margins": 2.5953850746154785,
      "rewards/rejected": -1.674921989440918,
      "step": 494
    },
    {
      "epoch": 0.26,
      "grad_norm": 12.930924815591139,
      "kl": 0.0,
      "learning_rate": 3.7048665620094186e-07,
      "logps/chosen": -273.54595947265625,
      "logps/rejected": -246.6627655029297,
      "loss": 0.2992,
      "rewards/chosen": -1.0437400341033936,
      "rewards/margins": 0.9890058040618896,
      "rewards/rejected": -2.032745838165283,
      "step": 495
    },
    {
      "epoch": 0.26,
      "grad_norm": 9.386730828304604,
      "kl": 0.0,
      "learning_rate": 3.702250130821559e-07,
      "logps/chosen": -225.7566680908203,
      "logps/rejected": -239.1521759033203,
      "loss": 0.1486,
      "rewards/chosen": 0.671124279499054,
      "rewards/margins": 3.3144171237945557,
      "rewards/rejected": -2.6432929039001465,
      "step": 496
    },
    {
      "epoch": 0.26,
      "grad_norm": 12.454753782186298,
      "kl": 0.0,
      "learning_rate": 3.6996336996336996e-07,
      "logps/chosen": -289.2835693359375,
      "logps/rejected": -271.740478515625,
      "loss": 0.2592,
      "rewards/chosen": -1.0335984230041504,
      "rewards/margins": 1.533452033996582,
      "rewards/rejected": -2.5670504570007324,
      "step": 497
    },
    {
      "epoch": 0.26,
      "grad_norm": 11.975373879855518,
      "kl": 0.0,
      "learning_rate": 3.6970172684458396e-07,
      "logps/chosen": -213.38905334472656,
      "logps/rejected": -201.7144012451172,
      "loss": 0.1521,
      "rewards/chosen": 0.3211534321308136,
      "rewards/margins": 1.6761234998703003,
      "rewards/rejected": -1.354970097541809,
      "step": 498
    },
    {
      "epoch": 0.26,
      "grad_norm": 10.925688675488383,
      "kl": 0.0,
      "learning_rate": 3.69440083725798e-07,
      "logps/chosen": -208.6687774658203,
      "logps/rejected": -211.62774658203125,
      "loss": 0.1478,
      "rewards/chosen": 1.2924268245697021,
      "rewards/margins": 3.0114760398864746,
      "rewards/rejected": -1.7190492153167725,
      "step": 499
    },
    {
      "epoch": 0.26,
      "grad_norm": 12.744456658603918,
      "kl": 0.0,
      "learning_rate": 3.69178440607012e-07,
      "logps/chosen": -195.3672332763672,
      "logps/rejected": -232.05824279785156,
      "loss": 0.273,
      "rewards/chosen": -0.07960432767868042,
      "rewards/margins": 2.1937575340270996,
      "rewards/rejected": -2.273361921310425,
      "step": 500
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 1911,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}