{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9997382884061764,
  "eval_steps": 100,
  "global_step": 955,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0010468463752944255,
      "grad_norm": 5.5,
      "learning_rate": 5.208333333333333e-08,
      "logits/chosen": -0.3494967222213745,
      "logits/rejected": -0.3728627860546112,
      "logps/chosen": -285.8127136230469,
      "logps/rejected": -212.7957000732422,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.010468463752944255,
      "grad_norm": 6.0,
      "learning_rate": 5.208333333333334e-07,
      "logits/chosen": -0.5402188301086426,
      "logits/rejected": -0.5500054955482483,
      "logps/chosen": -315.3569030761719,
      "logps/rejected": -278.0135192871094,
      "loss": 0.6971,
      "rewards/accuracies": 0.4236111044883728,
      "rewards/chosen": -0.0025262832641601562,
      "rewards/margins": -0.0020421938970685005,
      "rewards/rejected": -0.0004840891051571816,
      "step": 10
    },
    {
      "epoch": 0.02093692750588851,
      "grad_norm": 5.46875,
      "learning_rate": 1.0416666666666667e-06,
      "logits/chosen": -0.5040005445480347,
      "logits/rejected": -0.5246113538742065,
      "logps/chosen": -306.70074462890625,
      "logps/rejected": -271.21923828125,
      "loss": 0.6942,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.007291222922503948,
      "rewards/margins": 0.007475622929632664,
      "rewards/rejected": -0.00018439842096995562,
      "step": 20
    },
    {
      "epoch": 0.031405391258832765,
      "grad_norm": 5.9375,
      "learning_rate": 1.5625e-06,
      "logits/chosen": -0.5098769664764404,
      "logits/rejected": -0.5173646807670593,
      "logps/chosen": -291.0880432128906,
      "logps/rejected": -252.43124389648438,
      "loss": 0.6896,
      "rewards/accuracies": 0.5562499761581421,
      "rewards/chosen": 0.007684030570089817,
      "rewards/margins": 0.009358614683151245,
      "rewards/rejected": -0.001674586907029152,
      "step": 30
    },
    {
      "epoch": 0.04187385501177702,
      "grad_norm": 5.375,
      "learning_rate": 2.0833333333333334e-06,
      "logits/chosen": -0.4819061756134033,
      "logits/rejected": -0.5171386003494263,
      "logps/chosen": -306.039794921875,
      "logps/rejected": -244.5795135498047,
      "loss": 0.6816,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.026073966175317764,
      "rewards/margins": 0.02752821519970894,
      "rewards/rejected": -0.0014542475109919906,
      "step": 40
    },
    {
      "epoch": 0.05234231876472128,
      "grad_norm": 4.5625,
      "learning_rate": 2.604166666666667e-06,
      "logits/chosen": -0.5450257658958435,
      "logits/rejected": -0.5735383033752441,
      "logps/chosen": -305.06109619140625,
      "logps/rejected": -282.95440673828125,
      "loss": 0.6762,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.05157756805419922,
      "rewards/margins": 0.039979297667741776,
      "rewards/rejected": 0.011598266661167145,
      "step": 50
    },
    {
      "epoch": 0.06281078251766553,
      "grad_norm": 5.40625,
      "learning_rate": 3.125e-06,
      "logits/chosen": -0.5655590891838074,
      "logits/rejected": -0.5666534900665283,
      "logps/chosen": -290.9037780761719,
      "logps/rejected": -254.7635955810547,
      "loss": 0.6644,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": 0.08077142387628555,
      "rewards/margins": 0.037061307579278946,
      "rewards/rejected": 0.04371010884642601,
      "step": 60
    },
    {
      "epoch": 0.07327924627060979,
      "grad_norm": 5.0625,
      "learning_rate": 3.6458333333333333e-06,
      "logits/chosen": -0.5553646683692932,
      "logits/rejected": -0.5701892375946045,
      "logps/chosen": -286.6636962890625,
      "logps/rejected": -263.30523681640625,
      "loss": 0.6466,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.16360881924629211,
      "rewards/margins": 0.1136423796415329,
      "rewards/rejected": 0.04996642470359802,
      "step": 70
    },
    {
      "epoch": 0.08374771002355404,
      "grad_norm": 4.78125,
      "learning_rate": 4.166666666666667e-06,
      "logits/chosen": -0.4944031834602356,
      "logits/rejected": -0.5216720700263977,
      "logps/chosen": -287.98211669921875,
      "logps/rejected": -273.87139892578125,
      "loss": 0.6293,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.25989845395088196,
      "rewards/margins": 0.19782108068466187,
      "rewards/rejected": 0.06207737326622009,
      "step": 80
    },
    {
      "epoch": 0.0942161737764983,
      "grad_norm": 5.3125,
      "learning_rate": 4.6875000000000004e-06,
      "logits/chosen": -0.5244244337081909,
      "logits/rejected": -0.5471149682998657,
      "logps/chosen": -330.5142517089844,
      "logps/rejected": -295.2393798828125,
      "loss": 0.6177,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.2595275342464447,
      "rewards/margins": 0.25044435262680054,
      "rewards/rejected": 0.009083209559321404,
      "step": 90
    },
    {
      "epoch": 0.10468463752944256,
      "grad_norm": 5.71875,
      "learning_rate": 4.9997324926814375e-06,
      "logits/chosen": -0.5673449039459229,
      "logits/rejected": -0.5620633959770203,
      "logps/chosen": -275.5569763183594,
      "logps/rejected": -289.50457763671875,
      "loss": 0.6275,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.3272210955619812,
      "rewards/margins": 0.25637245178222656,
      "rewards/rejected": 0.07084863632917404,
      "step": 100
    },
    {
      "epoch": 0.10468463752944256,
      "eval_logits/chosen": -0.5618711709976196,
      "eval_logits/rejected": -0.5581737160682678,
      "eval_logps/chosen": -289.9532775878906,
      "eval_logps/rejected": -276.77532958984375,
      "eval_loss": 0.612218976020813,
      "eval_rewards/accuracies": 0.6919999718666077,
      "eval_rewards/chosen": 0.2593880295753479,
      "eval_rewards/margins": 0.2692645788192749,
      "eval_rewards/rejected": -0.00987652875483036,
      "eval_runtime": 351.8724,
      "eval_samples_per_second": 5.684,
      "eval_steps_per_second": 0.355,
      "step": 100
    },
    {
      "epoch": 0.11515310128238682,
      "grad_norm": 4.25,
      "learning_rate": 4.996723692767927e-06,
      "logits/chosen": -0.6067845225334167,
      "logits/rejected": -0.6330805420875549,
      "logps/chosen": -288.8458557128906,
      "logps/rejected": -276.23870849609375,
      "loss": 0.6078,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.27882444858551025,
      "rewards/margins": 0.34665778279304504,
      "rewards/rejected": -0.0678333044052124,
      "step": 110
    },
    {
      "epoch": 0.12562156503533106,
      "grad_norm": 5.0625,
      "learning_rate": 4.9903757462135984e-06,
      "logits/chosen": -0.5565638542175293,
      "logits/rejected": -0.5734590291976929,
      "logps/chosen": -260.9988098144531,
      "logps/rejected": -252.0515594482422,
      "loss": 0.591,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.15620097517967224,
      "rewards/margins": 0.29570046067237854,
      "rewards/rejected": -0.1394994556903839,
      "step": 120
    },
    {
      "epoch": 0.1360900287882753,
      "grad_norm": 5.5,
      "learning_rate": 4.980697142834315e-06,
      "logits/chosen": -0.5258967876434326,
      "logits/rejected": -0.5430957078933716,
      "logps/chosen": -301.23260498046875,
      "logps/rejected": -336.9810485839844,
      "loss": 0.5936,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.17174401879310608,
      "rewards/margins": 0.31216299533843994,
      "rewards/rejected": -0.14041896164417267,
      "step": 130
    },
    {
      "epoch": 0.14655849254121958,
      "grad_norm": 4.75,
      "learning_rate": 4.967700826904229e-06,
      "logits/chosen": -0.5994604825973511,
      "logits/rejected": -0.6112743616104126,
      "logps/chosen": -281.85223388671875,
      "logps/rejected": -274.6047058105469,
      "loss": 0.5781,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.12341972440481186,
      "rewards/margins": 0.4266698360443115,
      "rewards/rejected": -0.30325013399124146,
      "step": 140
    },
    {
      "epoch": 0.15702695629416383,
      "grad_norm": 4.8125,
      "learning_rate": 4.951404179843963e-06,
      "logits/chosen": -0.5988300442695618,
      "logits/rejected": -0.5718582272529602,
      "logps/chosen": -308.1268310546875,
      "logps/rejected": -279.3246765136719,
      "loss": 0.5775,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 0.22290559113025665,
      "rewards/margins": 0.45703545212745667,
      "rewards/rejected": -0.2341299057006836,
      "step": 150
    },
    {
      "epoch": 0.16749542004710807,
      "grad_norm": 4.5,
      "learning_rate": 4.931828996974498e-06,
      "logits/chosen": -0.5415905117988586,
      "logits/rejected": -0.5388587713241577,
      "logps/chosen": -297.1036682128906,
      "logps/rejected": -271.18475341796875,
      "loss": 0.5459,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.1913338154554367,
      "rewards/margins": 0.5190017819404602,
      "rewards/rejected": -0.3276679515838623,
      "step": 160
    },
    {
      "epoch": 0.17796388380005235,
      "grad_norm": 4.875,
      "learning_rate": 4.909001458367867e-06,
      "logits/chosen": -0.6261523365974426,
      "logits/rejected": -0.6151515245437622,
      "logps/chosen": -288.1965637207031,
      "logps/rejected": -275.8837585449219,
      "loss": 0.5809,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.018318424001336098,
      "rewards/margins": 0.4437525272369385,
      "rewards/rejected": -0.42543405294418335,
      "step": 170
    },
    {
      "epoch": 0.1884323475529966,
      "grad_norm": 5.15625,
      "learning_rate": 4.882952093833628e-06,
      "logits/chosen": -0.6297336220741272,
      "logits/rejected": -0.6054444313049316,
      "logps/chosen": -303.1052551269531,
      "logps/rejected": -265.9317932128906,
      "loss": 0.5599,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.07898494601249695,
      "rewards/margins": 0.4595085084438324,
      "rewards/rejected": -0.5384935140609741,
      "step": 180
    },
    {
      "epoch": 0.19890081130594087,
      "grad_norm": 5.875,
      "learning_rate": 4.853715742087947e-06,
      "logits/chosen": -0.5653724074363708,
      "logits/rejected": -0.5484523177146912,
      "logps/chosen": -276.7878723144531,
      "logps/rejected": -283.9636535644531,
      "loss": 0.5658,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": 0.021318156272172928,
      "rewards/margins": 0.4857168197631836,
      "rewards/rejected": -0.46439868211746216,
      "step": 190
    },
    {
      "epoch": 0.2093692750588851,
      "grad_norm": 6.5,
      "learning_rate": 4.821331504159906e-06,
      "logits/chosen": -0.5703824162483215,
      "logits/rejected": -0.5851908922195435,
      "logps/chosen": -298.11962890625,
      "logps/rejected": -256.55633544921875,
      "loss": 0.5726,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.06406483054161072,
      "rewards/margins": 0.5046994090080261,
      "rewards/rejected": -0.5687642097473145,
      "step": 200
    },
    {
      "epoch": 0.2093692750588851,
      "eval_logits/chosen": -0.5266345739364624,
      "eval_logits/rejected": -0.510294497013092,
      "eval_logps/chosen": -293.3344421386719,
      "eval_logps/rejected": -283.0293273925781,
      "eval_loss": 0.552874743938446,
      "eval_rewards/accuracies": 0.7039999961853027,
      "eval_rewards/chosen": -0.07872959226369858,
      "eval_rewards/margins": 0.5565416812896729,
      "eval_rewards/rejected": -0.6352713108062744,
      "eval_runtime": 349.2838,
      "eval_samples_per_second": 5.726,
      "eval_steps_per_second": 0.358,
      "step": 200
    },
    {
      "epoch": 0.21983773881182936,
      "grad_norm": 6.21875,
      "learning_rate": 4.7858426910973435e-06,
      "logits/chosen": -0.6147966980934143,
      "logits/rejected": -0.6168125867843628,
      "logps/chosen": -280.411865234375,
      "logps/rejected": -273.9303283691406,
      "loss": 0.5479,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.01772790588438511,
      "rewards/margins": 0.5174022912979126,
      "rewards/rejected": -0.5351301431655884,
      "step": 210
    },
    {
      "epoch": 0.23030620256477363,
      "grad_norm": 5.15625,
      "learning_rate": 4.747296766042161e-06,
      "logits/chosen": -0.5869513750076294,
      "logits/rejected": -0.5712444186210632,
      "logps/chosen": -319.57781982421875,
      "logps/rejected": -271.85662841796875,
      "loss": 0.5371,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.07761907577514648,
      "rewards/margins": 0.5722238421440125,
      "rewards/rejected": -0.49460476636886597,
      "step": 220
    },
    {
      "epoch": 0.24077466631771788,
      "grad_norm": 5.84375,
      "learning_rate": 4.705745280752586e-06,
      "logits/chosen": -0.6202965974807739,
      "logits/rejected": -0.5799211263656616,
      "logps/chosen": -292.62506103515625,
      "logps/rejected": -289.6524658203125,
      "loss": 0.5593,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.0319809652864933,
      "rewards/margins": 0.5654061436653137,
      "rewards/rejected": -0.5973870754241943,
      "step": 230
    },
    {
      "epoch": 0.2512431300706621,
      "grad_norm": 4.8125,
      "learning_rate": 4.661243806657256e-06,
      "logits/chosen": -0.5910071134567261,
      "logits/rejected": -0.5533290505409241,
      "logps/chosen": -300.802734375,
      "logps/rejected": -264.6828918457031,
      "loss": 0.5557,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.033292002975940704,
      "rewards/margins": 0.5110405087471008,
      "rewards/rejected": -0.5443325042724609,
      "step": 240
    },
    {
      "epoch": 0.26171159382360637,
      "grad_norm": 5.71875,
      "learning_rate": 4.613851860533367e-06,
      "logits/chosen": -0.6005420684814453,
      "logits/rejected": -0.5617684125900269,
      "logps/chosen": -294.0511169433594,
      "logps/rejected": -260.35443115234375,
      "loss": 0.5825,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.09464361518621445,
      "rewards/margins": 0.5028691291809082,
      "rewards/rejected": -0.40822547674179077,
      "step": 250
    },
    {
      "epoch": 0.2721800575765506,
      "grad_norm": 5.6875,
      "learning_rate": 4.563632824908252e-06,
      "logits/chosen": -0.5696755647659302,
      "logits/rejected": -0.5352145433425903,
      "logps/chosen": -293.357666015625,
      "logps/rejected": -279.78173828125,
      "loss": 0.5358,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.2980680465698242,
      "rewards/margins": 0.81305992603302,
      "rewards/rejected": -0.5149918794631958,
      "step": 260
    },
    {
      "epoch": 0.2826485213294949,
      "grad_norm": 5.59375,
      "learning_rate": 4.510653863290871e-06,
      "logits/chosen": -0.5663573741912842,
      "logits/rejected": -0.5443081259727478,
      "logps/chosen": -296.19744873046875,
      "logps/rejected": -303.76507568359375,
      "loss": 0.5498,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.06666271388530731,
      "rewards/margins": 0.5538051724433899,
      "rewards/rejected": -0.4871424734592438,
      "step": 270
    },
    {
      "epoch": 0.29311698508243916,
      "grad_norm": 4.8125,
      "learning_rate": 4.454985830346574e-06,
      "logits/chosen": -0.6322006583213806,
      "logits/rejected": -0.598243236541748,
      "logps/chosen": -301.9215087890625,
      "logps/rejected": -285.16412353515625,
      "loss": 0.5696,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": 0.03948987275362015,
      "rewards/margins": 0.49146270751953125,
      "rewards/rejected": -0.4519728720188141,
      "step": 280
    },
    {
      "epoch": 0.3035854488353834,
      "grad_norm": 4.65625,
      "learning_rate": 4.396703177135262e-06,
      "logits/chosen": -0.5908337831497192,
      "logits/rejected": -0.5626215934753418,
      "logps/chosen": -287.64849853515625,
      "logps/rejected": -258.74005126953125,
      "loss": 0.526,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.14304542541503906,
      "rewards/margins": 0.6406429409980774,
      "rewards/rejected": -0.49759751558303833,
      "step": 290
    },
    {
      "epoch": 0.31405391258832765,
      "grad_norm": 5.75,
      "learning_rate": 4.335883851539693e-06,
      "logits/chosen": -0.6105285882949829,
      "logits/rejected": -0.5751320123672485,
      "logps/chosen": -297.8828125,
      "logps/rejected": -294.6984558105469,
      "loss": 0.5429,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.1768026053905487,
      "rewards/margins": 0.6462854743003845,
      "rewards/rejected": -0.8230881690979004,
      "step": 300
    },
    {
      "epoch": 0.31405391258832765,
      "eval_logits/chosen": -0.4910103976726532,
      "eval_logits/rejected": -0.4688774645328522,
      "eval_logps/chosen": -294.2773132324219,
      "eval_logps/rejected": -285.1317138671875,
      "eval_loss": 0.5380240678787231,
      "eval_rewards/accuracies": 0.7260000109672546,
      "eval_rewards/chosen": -0.1730155199766159,
      "eval_rewards/margins": 0.6724972128868103,
      "eval_rewards/rejected": -0.8455128073692322,
      "eval_runtime": 349.3616,
      "eval_samples_per_second": 5.725,
      "eval_steps_per_second": 0.358,
      "step": 300
    },
    {
      "epoch": 0.3245223763412719,
      "grad_norm": 6.625,
      "learning_rate": 4.2726091940171055e-06,
      "logits/chosen": -0.5600444078445435,
      "logits/rejected": -0.5925548672676086,
      "logps/chosen": -296.61083984375,
      "logps/rejected": -342.74749755859375,
      "loss": 0.5077,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.023765895515680313,
      "rewards/margins": 0.7941875457763672,
      "rewards/rejected": -0.7704216241836548,
      "step": 310
    },
    {
      "epoch": 0.33499084009421615,
      "grad_norm": 5.15625,
      "learning_rate": 4.206963828813555e-06,
      "logits/chosen": -0.6118437647819519,
      "logits/rejected": -0.5762773752212524,
      "logps/chosen": -297.16278076171875,
      "logps/rejected": -280.4404602050781,
      "loss": 0.531,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.036954768002033234,
      "rewards/margins": 0.7416526079177856,
      "rewards/rejected": -0.7786073684692383,
      "step": 320
    },
    {
      "epoch": 0.34545930384716045,
      "grad_norm": 5.53125,
      "learning_rate": 4.139035550786495e-06,
      "logits/chosen": -0.6456430554389954,
      "logits/rejected": -0.5850483179092407,
      "logps/chosen": -290.33062744140625,
      "logps/rejected": -261.4778137207031,
      "loss": 0.5252,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.11032617092132568,
      "rewards/margins": 0.6117701530456543,
      "rewards/rejected": -0.7220962643623352,
      "step": 330
    },
    {
      "epoch": 0.3559277676001047,
      "grad_norm": 6.1875,
      "learning_rate": 4.068915207986931e-06,
      "logits/chosen": -0.6017712354660034,
      "logits/rejected": -0.5405057072639465,
      "logps/chosen": -298.4487609863281,
      "logps/rejected": -258.83343505859375,
      "loss": 0.5244,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.27785810828208923,
      "rewards/margins": 0.7664127945899963,
      "rewards/rejected": -1.0442708730697632,
      "step": 340
    },
    {
      "epoch": 0.36639623135304894,
      "grad_norm": 6.25,
      "learning_rate": 3.996696580158211e-06,
      "logits/chosen": -0.5467532873153687,
      "logits/rejected": -0.525019645690918,
      "logps/chosen": -337.39007568359375,
      "logps/rejected": -292.0143127441406,
      "loss": 0.5204,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.04514966905117035,
      "rewards/margins": 0.7053651809692383,
      "rewards/rejected": -0.7505148649215698,
      "step": 350
    },
    {
      "epoch": 0.3768646951059932,
      "grad_norm": 5.03125,
      "learning_rate": 3.922476253313921e-06,
      "logits/chosen": -0.5269440412521362,
      "logits/rejected": -0.5347733497619629,
      "logps/chosen": -275.55755615234375,
      "logps/rejected": -298.5218811035156,
      "loss": 0.5549,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.23508167266845703,
      "rewards/margins": 0.6767452955245972,
      "rewards/rejected": -0.9118270874023438,
      "step": 360
    },
    {
      "epoch": 0.38733315885893743,
      "grad_norm": 5.78125,
      "learning_rate": 3.846353490562664e-06,
      "logits/chosen": -0.5508509278297424,
      "logits/rejected": -0.5576980113983154,
      "logps/chosen": -290.8457946777344,
      "logps/rejected": -265.32647705078125,
      "loss": 0.508,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06573277711868286,
      "rewards/margins": 0.8556879758834839,
      "rewards/rejected": -0.9214208722114563,
      "step": 370
    },
    {
      "epoch": 0.39780162261188173,
      "grad_norm": 5.28125,
      "learning_rate": 3.768430099352445e-06,
      "logits/chosen": -0.5762262344360352,
      "logits/rejected": -0.5799992084503174,
      "logps/chosen": -306.72760009765625,
      "logps/rejected": -279.5985107421875,
      "loss": 0.5075,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.010313669219613075,
      "rewards/margins": 0.9469502568244934,
      "rewards/rejected": -0.9572639465332031,
      "step": 380
    },
    {
      "epoch": 0.408270086364826,
      "grad_norm": 6.40625,
      "learning_rate": 3.6888102953122307e-06,
      "logits/chosen": -0.6278566122055054,
      "logits/rejected": -0.593431830406189,
      "logps/chosen": -262.35601806640625,
      "logps/rejected": -263.1413879394531,
      "loss": 0.5558,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06627392768859863,
      "rewards/margins": 0.7061600089073181,
      "rewards/rejected": -0.639886200428009,
      "step": 390
    },
    {
      "epoch": 0.4187385501177702,
      "grad_norm": 5.21875,
      "learning_rate": 3.607600562872785e-06,
      "logits/chosen": -0.5806395411491394,
      "logits/rejected": -0.5490812659263611,
      "logps/chosen": -284.69036865234375,
      "logps/rejected": -275.1679992675781,
      "loss": 0.5054,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": 0.10525357723236084,
      "rewards/margins": 0.7453383803367615,
      "rewards/rejected": -0.6400848031044006,
      "step": 400
    },
    {
      "epoch": 0.4187385501177702,
      "eval_logits/chosen": -0.45346179604530334,
      "eval_logits/rejected": -0.42607197165489197,
      "eval_logps/chosen": -293.4172668457031,
      "eval_logps/rejected": -285.14593505859375,
      "eval_loss": 0.5331673622131348,
      "eval_rewards/accuracies": 0.7239999771118164,
      "eval_rewards/chosen": -0.08701039105653763,
      "eval_rewards/margins": 0.7599251866340637,
      "eval_rewards/rejected": -0.8469354510307312,
      "eval_runtime": 349.2048,
      "eval_samples_per_second": 5.727,
      "eval_steps_per_second": 0.358,
      "step": 400
    },
    {
      "epoch": 0.42920701387071447,
      "grad_norm": 5.1875,
      "learning_rate": 3.5249095128531863e-06,
      "logits/chosen": -0.6033052206039429,
      "logits/rejected": -0.5547356605529785,
      "logps/chosen": -279.104248046875,
      "logps/rejected": -276.4415588378906,
      "loss": 0.5154,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.09319617599248886,
      "rewards/margins": 0.780913770198822,
      "rewards/rejected": -0.8741100430488586,
      "step": 410
    },
    {
      "epoch": 0.4396754776236587,
      "grad_norm": 5.9375,
      "learning_rate": 3.4408477372034743e-06,
      "logits/chosen": -0.5911046266555786,
      "logits/rejected": -0.5682761669158936,
      "logps/chosen": -309.4401550292969,
      "logps/rejected": -297.2247009277344,
      "loss": 0.5621,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.05225636810064316,
      "rewards/margins": 0.636865496635437,
      "rewards/rejected": -0.6891219615936279,
      "step": 420
    },
    {
      "epoch": 0.45014394137660296,
      "grad_norm": 6.34375,
      "learning_rate": 3.355527661097728e-06,
      "logits/chosen": -0.5924620032310486,
      "logits/rejected": -0.595079243183136,
      "logps/chosen": -280.9834899902344,
      "logps/rejected": -281.77105712890625,
      "loss": 0.5226,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.19577598571777344,
      "rewards/margins": 0.608073353767395,
      "rewards/rejected": -0.8038493990898132,
      "step": 430
    },
    {
      "epoch": 0.46061240512954726,
      "grad_norm": 5.5625,
      "learning_rate": 3.269063392575352e-06,
      "logits/chosen": -0.5576502680778503,
      "logits/rejected": -0.5509397983551025,
      "logps/chosen": -328.8077087402344,
      "logps/rejected": -306.9642333984375,
      "loss": 0.5141,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.0049751936458051205,
      "rewards/margins": 0.7285463213920593,
      "rewards/rejected": -0.733521580696106,
      "step": 440
    },
    {
      "epoch": 0.4710808688824915,
      "grad_norm": 6.4375,
      "learning_rate": 3.181570569931697e-06,
      "logits/chosen": -0.5819513201713562,
      "logits/rejected": -0.5696187019348145,
      "logps/chosen": -287.6014709472656,
      "logps/rejected": -283.2913513183594,
      "loss": 0.5033,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.24654550850391388,
      "rewards/margins": 0.6834465265274048,
      "rewards/rejected": -0.9299919009208679,
      "step": 450
    },
    {
      "epoch": 0.48154933263543576,
      "grad_norm": 4.8125,
      "learning_rate": 3.09316620706208e-06,
      "logits/chosen": -0.5356208086013794,
      "logits/rejected": -0.5477502346038818,
      "logps/chosen": -308.5799560546875,
      "logps/rejected": -289.5464172363281,
      "loss": 0.493,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.07389375567436218,
      "rewards/margins": 0.9217998385429382,
      "rewards/rejected": -0.995693564414978,
      "step": 460
    },
    {
      "epoch": 0.49201779638838,
      "grad_norm": 5.3125,
      "learning_rate": 3.0039685369660785e-06,
      "logits/chosen": -0.5380591154098511,
      "logits/rejected": -0.49667781591415405,
      "logps/chosen": -283.11297607421875,
      "logps/rejected": -267.4956359863281,
      "loss": 0.5366,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.035861968994140625,
      "rewards/margins": 0.8068122863769531,
      "rewards/rejected": -0.7709503173828125,
      "step": 470
    },
    {
      "epoch": 0.5024862601413242,
      "grad_norm": 5.375,
      "learning_rate": 2.91409685362137e-06,
      "logits/chosen": -0.5487444996833801,
      "logits/rejected": -0.5391730666160583,
      "logps/chosen": -280.1451110839844,
      "logps/rejected": -277.56793212890625,
      "loss": 0.5096,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.06115647405385971,
      "rewards/margins": 0.7818087339401245,
      "rewards/rejected": -0.8429651260375977,
      "step": 480
    },
    {
      "epoch": 0.5129547238942685,
      "grad_norm": 4.8125,
      "learning_rate": 2.8236713524386085e-06,
      "logits/chosen": -0.6103276014328003,
      "logits/rejected": -0.5744163393974304,
      "logps/chosen": -279.46942138671875,
      "logps/rejected": -257.06500244140625,
      "loss": 0.5051,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14644916355609894,
      "rewards/margins": 0.8112751841545105,
      "rewards/rejected": -0.6648260354995728,
      "step": 490
    },
    {
      "epoch": 0.5234231876472127,
      "grad_norm": 5.5625,
      "learning_rate": 2.7328129695107205e-06,
      "logits/chosen": -0.5192793607711792,
      "logits/rejected": -0.5326711535453796,
      "logps/chosen": -264.8558349609375,
      "logps/rejected": -274.16455078125,
      "loss": 0.5508,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 0.14748027920722961,
      "rewards/margins": 0.9845119714736938,
      "rewards/rejected": -0.8370317220687866,
      "step": 500
    },
    {
      "epoch": 0.5234231876472127,
      "eval_logits/chosen": -0.43347951769828796,
      "eval_logits/rejected": -0.4044938385486603,
      "eval_logps/chosen": -292.7540283203125,
      "eval_logps/rejected": -284.7646179199219,
      "eval_loss": 0.5267207622528076,
      "eval_rewards/accuracies": 0.7179999947547913,
      "eval_rewards/chosen": -0.02068748138844967,
      "eval_rewards/margins": 0.7881156206130981,
      "eval_rewards/rejected": -0.8088030219078064,
      "eval_runtime": 349.1516,
      "eval_samples_per_second": 5.728,
      "eval_steps_per_second": 0.358,
      "step": 500
    },
    {
      "epoch": 0.533891651400157,
      "grad_norm": 4.65625,
      "learning_rate": 2.641643219871597e-06,
      "logits/chosen": -0.5523602962493896,
      "logits/rejected": -0.5135177969932556,
      "logps/chosen": -313.77227783203125,
      "logps/rejected": -297.81756591796875,
      "loss": 0.5053,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.082377590239048,
      "rewards/margins": 0.7289600968360901,
      "rewards/rejected": -0.8113376498222351,
      "step": 510
    },
    {
      "epoch": 0.5443601151531012,
      "grad_norm": 6.84375,
      "learning_rate": 2.5502840349805074e-06,
      "logits/chosen": -0.5441479086875916,
      "logits/rejected": -0.5340668559074402,
      "logps/chosen": -309.962158203125,
      "logps/rejected": -297.2099304199219,
      "loss": 0.5476,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.005710450001060963,
      "rewards/margins": 0.8396749496459961,
      "rewards/rejected": -0.8339645266532898,
      "step": 520
    },
    {
      "epoch": 0.5548285789060455,
      "grad_norm": 5.125,
      "learning_rate": 2.4588575996495797e-06,
      "logits/chosen": -0.4979037344455719,
      "logits/rejected": -0.5032162666320801,
      "logps/chosen": -272.52459716796875,
      "logps/rejected": -262.5303039550781,
      "loss": 0.5183,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.2069997489452362,
      "rewards/margins": 0.838523268699646,
      "rewards/rejected": -1.0455230474472046,
      "step": 530
    },
    {
      "epoch": 0.5652970426589898,
      "grad_norm": 5.59375,
      "learning_rate": 2.367486188632446e-06,
      "logits/chosen": -0.5421893000602722,
      "logits/rejected": -0.539043128490448,
      "logps/chosen": -286.2185363769531,
      "logps/rejected": -325.55206298828125,
      "loss": 0.5085,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.05336964130401611,
      "rewards/margins": 0.8456710577011108,
      "rewards/rejected": -0.8990408182144165,
      "step": 540
    },
    {
      "epoch": 0.575765506411934,
      "grad_norm": 4.34375,
      "learning_rate": 2.276292003092593e-06,
      "logits/chosen": -0.5617792010307312,
      "logits/rejected": -0.5426347851753235,
      "logps/chosen": -257.9263610839844,
      "logps/rejected": -265.53057861328125,
      "loss": 0.486,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.004675072617828846,
      "rewards/margins": 0.9126423597335815,
      "rewards/rejected": -0.9173175096511841,
      "step": 550
    },
    {
      "epoch": 0.5862339701648783,
      "grad_norm": 4.84375,
      "learning_rate": 2.1853970071701415e-06,
      "logits/chosen": -0.5527303218841553,
      "logits/rejected": -0.5213286876678467,
      "logps/chosen": -279.15924072265625,
      "logps/rejected": -280.146484375,
      "loss": 0.5179,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.01912788115441799,
      "rewards/margins": 0.8087967038154602,
      "rewards/rejected": -0.7896687388420105,
      "step": 560
    },
    {
      "epoch": 0.5967024339178225,
      "grad_norm": 5.71875,
      "learning_rate": 2.0949227648656194e-06,
      "logits/chosen": -0.5781675577163696,
      "logits/rejected": -0.5574057102203369,
      "logps/chosen": -295.38726806640625,
      "logps/rejected": -262.20684814453125,
      "loss": 0.5412,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.023471500724554062,
      "rewards/margins": 0.8612861633300781,
      "rewards/rejected": -0.8847578167915344,
      "step": 570
    },
    {
      "epoch": 0.6071708976707668,
      "grad_norm": 5.8125,
      "learning_rate": 2.00499027745888e-06,
      "logits/chosen": -0.5529965162277222,
      "logits/rejected": -0.5318363308906555,
      "logps/chosen": -299.06695556640625,
      "logps/rejected": -297.82098388671875,
      "loss": 0.5449,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": 0.07610292732715607,
      "rewards/margins": 0.7717056274414062,
      "rewards/rejected": -0.6956026554107666,
      "step": 580
    },
    {
      "epoch": 0.6176393614237111,
      "grad_norm": 5.625,
      "learning_rate": 1.915719821680624e-06,
      "logits/chosen": -0.5771445035934448,
      "logits/rejected": -0.5279287099838257,
      "logps/chosen": -286.1835021972656,
      "logps/rejected": -281.80828857421875,
      "loss": 0.5206,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.37562283873558044,
      "rewards/margins": 0.8612510561943054,
      "rewards/rejected": -0.485628217458725,
      "step": 590
    },
    {
      "epoch": 0.6281078251766553,
      "grad_norm": 5.75,
      "learning_rate": 1.8272307888529276e-06,
      "logits/chosen": -0.5050173997879028,
      "logits/rejected": -0.4642249643802643,
      "logps/chosen": -262.1390380859375,
      "logps/rejected": -278.6227111816406,
      "loss": 0.5338,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.2233264446258545,
      "rewards/margins": 0.8637221455574036,
      "rewards/rejected": -0.6403957009315491,
      "step": 600
    },
    {
      "epoch": 0.6281078251766553,
      "eval_logits/chosen": -0.4303784966468811,
      "eval_logits/rejected": -0.40024831891059875,
      "eval_logps/chosen": -290.5658874511719,
      "eval_logps/rejected": -282.5771179199219,
      "eval_loss": 0.5262989401817322,
      "eval_rewards/accuracies": 0.7300000190734863,
      "eval_rewards/chosen": 0.19812585413455963,
      "eval_rewards/margins": 0.788177490234375,
      "eval_rewards/rejected": -0.5900516510009766,
      "eval_runtime": 349.0523,
      "eval_samples_per_second": 5.73,
      "eval_steps_per_second": 0.358,
      "step": 600
    },
    {
      "epoch": 0.6385762889295996,
      "grad_norm": 4.53125,
      "learning_rate": 1.739641525213929e-06,
      "logits/chosen": -0.5362275838851929,
      "logits/rejected": -0.5250522494316101,
      "logps/chosen": -264.38677978515625,
      "logps/rejected": -269.70172119140625,
      "loss": 0.4947,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.2012002021074295,
      "rewards/margins": 0.8771921396255493,
      "rewards/rejected": -0.675991952419281,
      "step": 610
    },
    {
      "epoch": 0.6490447526825438,
      "grad_norm": 4.65625,
      "learning_rate": 1.6530691736402317e-06,
      "logits/chosen": -0.547565221786499,
      "logits/rejected": -0.5210016965866089,
      "logps/chosen": -292.267822265625,
      "logps/rejected": -281.25054931640625,
      "loss": 0.5076,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.06052354723215103,
      "rewards/margins": 0.8467457890510559,
      "rewards/rejected": -0.7862221598625183,
      "step": 620
    },
    {
      "epoch": 0.6595132164354881,
      "grad_norm": 5.5625,
      "learning_rate": 1.5676295169786864e-06,
      "logits/chosen": -0.5603654980659485,
      "logits/rejected": -0.5221408605575562,
      "logps/chosen": -287.28192138671875,
      "logps/rejected": -272.3338317871094,
      "loss": 0.5266,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.08671960234642029,
      "rewards/margins": 0.8218274116516113,
      "rewards/rejected": -0.908547043800354,
      "step": 630
    },
    {
      "epoch": 0.6699816801884323,
      "grad_norm": 5.40625,
      "learning_rate": 1.4834368231970922e-06,
      "logits/chosen": -0.5882564783096313,
      "logits/rejected": -0.5383986234664917,
      "logps/chosen": -288.31964111328125,
      "logps/rejected": -273.548583984375,
      "loss": 0.5013,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.17558792233467102,
      "rewards/margins": 0.7076612710952759,
      "rewards/rejected": -0.8832491040229797,
      "step": 640
    },
    {
      "epoch": 0.6804501439413766,
      "grad_norm": 5.3125,
      "learning_rate": 1.4006036925609245e-06,
      "logits/chosen": -0.5553715825080872,
      "logits/rejected": -0.5215333104133606,
      "logps/chosen": -301.14434814453125,
      "logps/rejected": -250.81289672851562,
      "loss": 0.539,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.25257399678230286,
      "rewards/margins": 0.822566032409668,
      "rewards/rejected": -1.0751399993896484,
      "step": 650
    },
    {
      "epoch": 0.6909186076943209,
      "grad_norm": 5.1875,
      "learning_rate": 1.3192409070404582e-06,
      "logits/chosen": -0.5745340585708618,
      "logits/rejected": -0.551754355430603,
      "logps/chosen": -305.2149658203125,
      "logps/rejected": -307.12396240234375,
      "loss": 0.5097,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.0797838568687439,
      "rewards/margins": 0.7875758409500122,
      "rewards/rejected": -0.8673597574234009,
      "step": 660
    },
    {
      "epoch": 0.7013870714472651,
      "grad_norm": 6.6875,
      "learning_rate": 1.2394572821496953e-06,
      "logits/chosen": -0.5750129818916321,
      "logits/rejected": -0.5432807207107544,
      "logps/chosen": -278.31634521484375,
      "logps/rejected": -259.3590087890625,
      "loss": 0.5169,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.12945926189422607,
      "rewards/margins": 0.7823888659477234,
      "rewards/rejected": -0.9118480682373047,
      "step": 670
    },
    {
      "epoch": 0.7118555352002094,
      "grad_norm": 4.9375,
      "learning_rate": 1.1613595214152713e-06,
      "logits/chosen": -0.6039143800735474,
      "logits/rejected": -0.5614811182022095,
      "logps/chosen": -288.0231628417969,
      "logps/rejected": -276.5443420410156,
      "loss": 0.5014,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.20713257789611816,
      "rewards/margins": 0.7958000302314758,
      "rewards/rejected": -1.0029326677322388,
      "step": 680
    },
    {
      "epoch": 0.7223239989531536,
      "grad_norm": 4.1875,
      "learning_rate": 1.0850520736699362e-06,
      "logits/chosen": -0.5435994863510132,
      "logits/rejected": -0.5199321508407593,
      "logps/chosen": -342.0343933105469,
      "logps/rejected": -316.83929443359375,
      "loss": 0.5367,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.19384647905826569,
      "rewards/margins": 0.8790246248245239,
      "rewards/rejected": -1.072871208190918,
      "step": 690
    },
    {
      "epoch": 0.7327924627060979,
      "grad_norm": 5.53125,
      "learning_rate": 1.0106369933615043e-06,
      "logits/chosen": -0.5901396870613098,
      "logits/rejected": -0.551986813545227,
      "logps/chosen": -316.48187255859375,
      "logps/rejected": -263.9020080566406,
      "loss": 0.5064,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.20547008514404297,
      "rewards/margins": 0.7208881378173828,
      "rewards/rejected": -0.9263582229614258,
      "step": 700
    },
    {
      "epoch": 0.7327924627060979,
      "eval_logits/chosen": -0.4080447256565094,
      "eval_logits/rejected": -0.37605100870132446,
      "eval_logps/chosen": -294.5545959472656,
      "eval_logps/rejected": -286.7521057128906,
      "eval_loss": 0.5174716114997864,
      "eval_rewards/accuracies": 0.7300000190734863,
      "eval_rewards/chosen": -0.20074398815631866,
      "eval_rewards/margins": 0.8068088889122009,
      "eval_rewards/rejected": -1.0075528621673584,
      "eval_runtime": 349.1622,
      "eval_samples_per_second": 5.728,
      "eval_steps_per_second": 0.358,
      "step": 700
    },
    {
      "epoch": 0.7432609264590422,
      "grad_norm": 6.03125,
      "learning_rate": 9.382138040640714e-07,
      "logits/chosen": -0.5988644361495972,
      "logits/rejected": -0.5565031170845032,
      "logps/chosen": -265.79888916015625,
      "logps/rejected": -279.4545593261719,
      "loss": 0.5573,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.14814189076423645,
      "rewards/margins": 0.8269087076187134,
      "rewards/rejected": -0.9750504493713379,
      "step": 710
    },
    {
      "epoch": 0.7537293902119864,
      "grad_norm": 5.3125,
      "learning_rate": 8.678793653740633e-07,
      "logits/chosen": -0.5240329504013062,
      "logits/rejected": -0.522124171257019,
      "logps/chosen": -264.15338134765625,
      "logps/rejected": -263.60552978515625,
      "loss": 0.5183,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0942729264497757,
      "rewards/margins": 0.8182271122932434,
      "rewards/rejected": -0.9125000834465027,
      "step": 720
    },
    {
      "epoch": 0.7641978539649307,
      "grad_norm": 4.6875,
      "learning_rate": 7.997277433690984e-07,
      "logits/chosen": -0.5428770780563354,
      "logits/rejected": -0.4947339594364166,
      "logps/chosen": -302.0629577636719,
      "logps/rejected": -287.492431640625,
      "loss": 0.5134,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.12038590759038925,
      "rewards/margins": 0.7666383981704712,
      "rewards/rejected": -0.8870244026184082,
      "step": 730
    },
    {
      "epoch": 0.7746663177178749,
      "grad_norm": 4.6875,
      "learning_rate": 7.338500848029603e-07,
      "logits/chosen": -0.49188748002052307,
      "logits/rejected": -0.5088191628456116,
      "logps/chosen": -292.02117919921875,
      "logps/rejected": -275.2965393066406,
      "loss": 0.4939,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.023575307801365852,
      "rewards/margins": 0.7873858213424683,
      "rewards/rejected": -0.7638105154037476,
      "step": 740
    },
    {
      "epoch": 0.7851347814708192,
      "grad_norm": 5.0625,
      "learning_rate": 6.70334495204884e-07,
      "logits/chosen": -0.5374957919120789,
      "logits/rejected": -0.516038179397583,
      "logps/chosen": -324.4206848144531,
      "logps/rejected": -286.3302917480469,
      "loss": 0.5036,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.0536157488822937,
      "rewards/margins": 0.7997394800186157,
      "rewards/rejected": -0.746123731136322,
      "step": 750
    },
    {
      "epoch": 0.7956032452237635,
      "grad_norm": 5.34375,
      "learning_rate": 6.092659210462232e-07,
      "logits/chosen": -0.5625780820846558,
      "logits/rejected": -0.5423771142959595,
      "logps/chosen": -269.4684143066406,
      "logps/rejected": -268.89886474609375,
      "loss": 0.5496,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.04108331725001335,
      "rewards/margins": 0.7257132530212402,
      "rewards/rejected": -0.7667967081069946,
      "step": 760
    },
    {
      "epoch": 0.8060717089767077,
      "grad_norm": 5.6875,
      "learning_rate": 5.507260361320738e-07,
      "logits/chosen": -0.5479773283004761,
      "logits/rejected": -0.5508543252944946,
      "logps/chosen": -285.4389953613281,
      "logps/rejected": -279.5918884277344,
      "loss": 0.5066,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.08319403976202011,
      "rewards/margins": 0.6797414422035217,
      "rewards/rejected": -0.5965473651885986,
      "step": 770
    },
    {
      "epoch": 0.816540172729652,
      "grad_norm": 5.21875,
      "learning_rate": 4.947931323697983e-07,
      "logits/chosen": -0.5364648103713989,
      "logits/rejected": -0.5092785954475403,
      "logps/chosen": -286.8380432128906,
      "logps/rejected": -279.98956298828125,
      "loss": 0.522,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.04336369037628174,
      "rewards/margins": 0.6655139327049255,
      "rewards/rejected": -0.7088775634765625,
      "step": 780
    },
    {
      "epoch": 0.8270086364825961,
      "grad_norm": 5.4375,
      "learning_rate": 4.4154201506053985e-07,
      "logits/chosen": -0.5611253976821899,
      "logits/rejected": -0.5496819019317627,
      "logps/chosen": -301.17742919921875,
      "logps/rejected": -264.5899658203125,
      "loss": 0.5154,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.043927911669015884,
      "rewards/margins": 0.7878991365432739,
      "rewards/rejected": -0.8318269848823547,
      "step": 790
    },
    {
      "epoch": 0.8374771002355405,
      "grad_norm": 4.875,
      "learning_rate": 3.910439028537638e-07,
      "logits/chosen": -0.5692980289459229,
      "logits/rejected": -0.5227521657943726,
      "logps/chosen": -348.53631591796875,
      "logps/rejected": -302.92352294921875,
      "loss": 0.5349,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": 0.09300284087657928,
      "rewards/margins": 0.6956671476364136,
      "rewards/rejected": -0.6026642322540283,
      "step": 800
    },
    {
      "epoch": 0.8374771002355405,
      "eval_logits/chosen": -0.4160803556442261,
      "eval_logits/rejected": -0.3852657377719879,
      "eval_logps/chosen": -292.3984069824219,
      "eval_logps/rejected": -284.57269287109375,
      "eval_loss": 0.5197243690490723,
      "eval_rewards/accuracies": 0.7200000286102295,
      "eval_rewards/chosen": 0.014875907450914383,
      "eval_rewards/margins": 0.8044845461845398,
      "eval_rewards/rejected": -0.7896087169647217,
      "eval_runtime": 349.1752,
      "eval_samples_per_second": 5.728,
      "eval_steps_per_second": 0.358,
      "step": 800
    },
    {
      "epoch": 0.8479455639884846,
      "grad_norm": 4.8125,
      "learning_rate": 3.4336633249862084e-07,
      "logits/chosen": -0.5947756767272949,
      "logits/rejected": -0.5246752500534058,
      "logps/chosen": -320.2247009277344,
      "logps/rejected": -289.26373291015625,
      "loss": 0.5012,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": 0.009494239464402199,
      "rewards/margins": 0.8274568319320679,
      "rewards/rejected": -0.8179625272750854,
      "step": 810
    },
    {
      "epoch": 0.8584140277414289,
      "grad_norm": 5.09375,
      "learning_rate": 2.98573068519539e-07,
      "logits/chosen": -0.570965588092804,
      "logits/rejected": -0.5559743046760559,
      "logps/chosen": -308.2550048828125,
      "logps/rejected": -294.05462646484375,
      "loss": 0.5287,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": 0.03936102241277695,
      "rewards/margins": 0.7610028982162476,
      "rewards/rejected": -0.7216418981552124,
      "step": 820
    },
    {
      "epoch": 0.8688824914943732,
      "grad_norm": 5.53125,
      "learning_rate": 2.5672401793681854e-07,
      "logits/chosen": -0.5910287499427795,
      "logits/rejected": -0.5737439393997192,
      "logps/chosen": -275.55743408203125,
      "logps/rejected": -270.0027770996094,
      "loss": 0.5101,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.0218903087079525,
      "rewards/margins": 0.816811203956604,
      "rewards/rejected": -0.8387015461921692,
      "step": 830
    },
    {
      "epoch": 0.8793509552473174,
      "grad_norm": 4.9375,
      "learning_rate": 2.178751501463036e-07,
      "logits/chosen": -0.5559400916099548,
      "logits/rejected": -0.5404902696609497,
      "logps/chosen": -315.4130859375,
      "logps/rejected": -308.131591796875,
      "loss": 0.5091,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.04114828258752823,
      "rewards/margins": 0.7911642789840698,
      "rewards/rejected": -0.8323125839233398,
      "step": 840
    },
    {
      "epoch": 0.8898194190002617,
      "grad_norm": 5.4375,
      "learning_rate": 1.820784220652766e-07,
      "logits/chosen": -0.6067087054252625,
      "logits/rejected": -0.5707641839981079,
      "logps/chosen": -347.02484130859375,
      "logps/rejected": -280.36297607421875,
      "loss": 0.5052,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.15417176485061646,
      "rewards/margins": 0.8600176572799683,
      "rewards/rejected": -0.7058458924293518,
      "step": 850
    },
    {
      "epoch": 0.9002878827532059,
      "grad_norm": 5.46875,
      "learning_rate": 1.4938170864468636e-07,
      "logits/chosen": -0.5292789936065674,
      "logits/rejected": -0.509742021560669,
      "logps/chosen": -290.58807373046875,
      "logps/rejected": -269.5320739746094,
      "loss": 0.4873,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.0017050117021426558,
      "rewards/margins": 0.8745778799057007,
      "rewards/rejected": -0.8728728294372559,
      "step": 860
    },
    {
      "epoch": 0.9107563465061502,
      "grad_norm": 5.625,
      "learning_rate": 1.1982873884064466e-07,
      "logits/chosen": -0.49922800064086914,
      "logits/rejected": -0.4975685477256775,
      "logps/chosen": -288.2568054199219,
      "logps/rejected": -277.89642333984375,
      "loss": 0.5241,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.032976943999528885,
      "rewards/margins": 0.6596282720565796,
      "rewards/rejected": -0.6926053166389465,
      "step": 870
    },
    {
      "epoch": 0.9212248102590945,
      "grad_norm": 5.78125,
      "learning_rate": 9.345903713082305e-08,
      "logits/chosen": -0.58247309923172,
      "logits/rejected": -0.574453592300415,
      "logps/chosen": -315.4455871582031,
      "logps/rejected": -281.5334167480469,
      "loss": 0.5391,
      "rewards/accuracies": 0.6312500238418579,
      "rewards/chosen": 0.010342386551201344,
      "rewards/margins": 0.6320812702178955,
      "rewards/rejected": -0.621738851070404,
      "step": 880
    },
    {
      "epoch": 0.9316932740120387,
      "grad_norm": 5.0625,
      "learning_rate": 7.030787065396866e-08,
      "logits/chosen": -0.5529004335403442,
      "logits/rejected": -0.5129944682121277,
      "logps/chosen": -319.9122314453125,
      "logps/rejected": -294.0019836425781,
      "loss": 0.5227,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.045671284198760986,
      "rewards/margins": 0.709865391254425,
      "rewards/rejected": -0.6641942262649536,
      "step": 890
    },
    {
      "epoch": 0.942161737764983,
      "grad_norm": 5.09375,
      "learning_rate": 5.0406202043228604e-08,
      "logits/chosen": -0.5645865201950073,
      "logits/rejected": -0.5604526400566101,
      "logps/chosen": -334.4140930175781,
      "logps/rejected": -275.6079406738281,
      "loss": 0.4775,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.1654743105173111,
      "rewards/margins": 0.9833230972290039,
      "rewards/rejected": -0.817848801612854,
      "step": 900
    },
    {
      "epoch": 0.942161737764983,
      "eval_logits/chosen": -0.4151337742805481,
      "eval_logits/rejected": -0.3841632902622223,
      "eval_logps/chosen": -292.3968200683594,
      "eval_logps/rejected": -284.6649169921875,
      "eval_loss": 0.5181210041046143,
      "eval_rewards/accuracies": 0.7260000109672546,
      "eval_rewards/chosen": 0.01503300666809082,
      "eval_rewards/margins": 0.8138652443885803,
      "eval_rewards/rejected": -0.7988322973251343,
      "eval_runtime": 349.0965,
      "eval_samples_per_second": 5.729,
      "eval_steps_per_second": 0.358,
      "step": 900
    },
    {
      "epoch": 0.9526302015179272,
      "grad_norm": 5.0625,
      "learning_rate": 3.378064801637687e-08,
      "logits/chosen": -0.6055993437767029,
      "logits/rejected": -0.557610034942627,
      "logps/chosen": -315.5115966796875,
      "logps/rejected": -314.4410400390625,
      "loss": 0.5209,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": 0.11909689754247665,
      "rewards/margins": 0.7703868746757507,
      "rewards/rejected": -0.6512900590896606,
      "step": 910
    },
    {
      "epoch": 0.9630986652708715,
      "grad_norm": 4.6875,
      "learning_rate": 2.0453443778310766e-08,
      "logits/chosen": -0.5314048528671265,
      "logits/rejected": -0.49928751587867737,
      "logps/chosen": -330.26824951171875,
      "logps/rejected": -305.7253723144531,
      "loss": 0.533,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.07462009787559509,
      "rewards/margins": 0.8954026103019714,
      "rewards/rejected": -0.820782482624054,
      "step": 920
    },
    {
      "epoch": 0.9735671290238157,
      "grad_norm": 4.78125,
      "learning_rate": 1.0442413283435759e-08,
      "logits/chosen": -0.5311998724937439,
      "logits/rejected": -0.49351367354393005,
      "logps/chosen": -318.3543701171875,
      "logps/rejected": -278.8912048339844,
      "loss": 0.514,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.13066162168979645,
      "rewards/margins": 1.1091994047164917,
      "rewards/rejected": -0.9785377383232117,
      "step": 930
    },
    {
      "epoch": 0.98403559277676,
      "grad_norm": 4.28125,
      "learning_rate": 3.760945397705828e-09,
      "logits/chosen": -0.5610886812210083,
      "logits/rejected": -0.5083616971969604,
      "logps/chosen": -291.5441589355469,
      "logps/rejected": -261.2994079589844,
      "loss": 0.4851,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": 0.18183191120624542,
      "rewards/margins": 1.0001600980758667,
      "rewards/rejected": -0.8183282613754272,
      "step": 940
    },
    {
      "epoch": 0.9945040565297043,
      "grad_norm": 4.875,
      "learning_rate": 4.1797599220405605e-10,
      "logits/chosen": -0.576126217842102,
      "logits/rejected": -0.5575291514396667,
      "logps/chosen": -295.9472961425781,
      "logps/rejected": -279.5827331542969,
      "loss": 0.5083,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11896193027496338,
      "rewards/margins": 0.8570976257324219,
      "rewards/rejected": -0.7381356358528137,
      "step": 950
    },
    {
      "epoch": 0.9997382884061764,
      "step": 955,
      "total_flos": 0.0,
      "train_loss": 0.5431846207973221,
      "train_runtime": 19104.9346,
      "train_samples_per_second": 3.2,
      "train_steps_per_second": 0.05
    }
  ],
  "logging_steps": 10,
  "max_steps": 955,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100000,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}