{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9973149213655543,
  "eval_steps": 50,
  "global_step": 1300,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.007671653241273495,
      "grad_norm": 1.4047255516052246,
      "learning_rate": 4.9616270145817344e-05,
      "logits/chosen": -0.5156614184379578,
      "logits/rejected": -0.5155973434448242,
      "logps/chosen": -1.6896766424179077,
      "logps/rejected": -3.5133140087127686,
      "loss": 0.6952,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00019091162539552897,
      "rewards/margins": 0.001644002622924745,
      "rewards/rejected": -0.0014530897606164217,
      "step": 10
    },
    {
      "epoch": 0.01534330648254699,
      "grad_norm": 1.5815409421920776,
      "learning_rate": 4.923254029163469e-05,
      "logits/chosen": -0.5282754302024841,
      "logits/rejected": -0.5348799824714661,
      "logps/chosen": -2.51069974899292,
      "logps/rejected": -2.873567819595337,
      "loss": 0.6911,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.003554452210664749,
      "rewards/margins": 0.004838407970964909,
      "rewards/rejected": -0.0012839583214372396,
      "step": 20
    },
    {
      "epoch": 0.023014959723820484,
      "grad_norm": 1.5293035507202148,
      "learning_rate": 4.8848810437452035e-05,
      "logits/chosen": -0.46481943130493164,
      "logits/rejected": -0.4620364308357239,
      "logps/chosen": -2.9464457035064697,
      "logps/rejected": -4.222188472747803,
      "loss": 0.6923,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.02544591762125492,
      "rewards/margins": 0.06909430772066116,
      "rewards/rejected": -0.09454023092985153,
      "step": 30
    },
    {
      "epoch": 0.03068661296509398,
      "grad_norm": 2.4810216426849365,
      "learning_rate": 4.846508058326938e-05,
      "logits/chosen": -0.4796638488769531,
      "logits/rejected": -0.4740970730781555,
      "logps/chosen": -4.077430725097656,
      "logps/rejected": -2.8998451232910156,
      "loss": 0.6951,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0723290666937828,
      "rewards/margins": -0.027712786570191383,
      "rewards/rejected": -0.044616274535655975,
      "step": 40
    },
    {
      "epoch": 0.038358266206367474,
      "grad_norm": 1.7016780376434326,
      "learning_rate": 4.8081350729086726e-05,
      "logits/chosen": -0.4345604479312897,
      "logits/rejected": -0.4407508969306946,
      "logps/chosen": -2.6753084659576416,
      "logps/rejected": -3.9175124168395996,
      "loss": 0.6948,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.022549040615558624,
      "rewards/margins": 0.023919526487588882,
      "rewards/rejected": -0.046468570828437805,
      "step": 50
    },
    {
      "epoch": 0.038358266206367474,
      "eval_logits/chosen": -0.2606842815876007,
      "eval_logits/rejected": -0.2579291760921478,
      "eval_logps/chosen": -3.532228946685791,
      "eval_logps/rejected": -3.719329833984375,
      "eval_loss": 0.6931471824645996,
      "eval_rewards/accuracies": 0.47187501192092896,
      "eval_rewards/chosen": -0.04993545264005661,
      "eval_rewards/margins": -0.011583473533391953,
      "eval_rewards/rejected": -0.03835199028253555,
      "eval_runtime": 92.1681,
      "eval_samples_per_second": 27.71,
      "eval_steps_per_second": 0.868,
      "step": 50
    },
    {
      "epoch": 0.04602991944764097,
      "grad_norm": 1.5512512922286987,
      "learning_rate": 4.769762087490407e-05,
      "logits/chosen": -0.41122546792030334,
      "logits/rejected": -0.4166368544101715,
      "logps/chosen": -3.745007276535034,
      "logps/rejected": -2.612656593322754,
      "loss": 0.6999,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.02822841703891754,
      "rewards/margins": -0.011006011627614498,
      "rewards/rejected": -0.01722240075469017,
      "step": 60
    },
    {
      "epoch": 0.05370157268891446,
      "grad_norm": 1.307566523551941,
      "learning_rate": 4.731389102072141e-05,
      "logits/chosen": -0.45976758003234863,
      "logits/rejected": -0.4562222361564636,
      "logps/chosen": -2.7820112705230713,
      "logps/rejected": -3.3871638774871826,
      "loss": 0.694,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.015000693500041962,
      "rewards/margins": 0.0025637741200625896,
      "rewards/rejected": -0.01756446808576584,
      "step": 70
    },
    {
      "epoch": 0.06137322593018796,
      "grad_norm": 1.6361109018325806,
      "learning_rate": 4.693016116653876e-05,
      "logits/chosen": -0.3749166429042816,
      "logits/rejected": -0.37954169511795044,
      "logps/chosen": -2.275099992752075,
      "logps/rejected": -4.181614398956299,
      "loss": 0.6907,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.020970512181520462,
      "rewards/margins": 0.025806229561567307,
      "rewards/rejected": -0.04677674546837807,
      "step": 80
    },
    {
      "epoch": 0.06904487917146145,
      "grad_norm": 1.83301842212677,
      "learning_rate": 4.65464313123561e-05,
      "logits/chosen": -0.5111284255981445,
      "logits/rejected": -0.5044411420822144,
      "logps/chosen": -2.0928683280944824,
      "logps/rejected": -5.220086574554443,
      "loss": 0.6899,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.017647314816713333,
      "rewards/margins": 0.07800659537315369,
      "rewards/rejected": -0.09565390646457672,
      "step": 90
    },
    {
      "epoch": 0.07671653241273495,
      "grad_norm": 2.6800460815429688,
      "learning_rate": 4.6162701458173444e-05,
      "logits/chosen": -0.44164401292800903,
      "logits/rejected": -0.44124874472618103,
      "logps/chosen": -3.620197296142578,
      "logps/rejected": -3.795258045196533,
      "loss": 0.7065,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.06498938798904419,
      "rewards/margins": 0.0013415880966931581,
      "rewards/rejected": -0.06633097678422928,
      "step": 100
    },
    {
      "epoch": 0.07671653241273495,
      "eval_logits/chosen": -0.25962817668914795,
      "eval_logits/rejected": -0.25671660900115967,
      "eval_logps/chosen": -3.3411738872528076,
      "eval_logps/rejected": -3.666893720626831,
      "eval_loss": 0.6914116144180298,
      "eval_rewards/accuracies": 0.53125,
      "eval_rewards/chosen": -0.030829980969429016,
      "eval_rewards/margins": 0.002278365660458803,
      "eval_rewards/rejected": -0.03310834616422653,
      "eval_runtime": 87.6826,
      "eval_samples_per_second": 29.128,
      "eval_steps_per_second": 0.912,
      "step": 100
    },
    {
      "epoch": 0.08438818565400844,
      "grad_norm": 1.4465539455413818,
      "learning_rate": 4.577897160399079e-05,
      "logits/chosen": -0.32688966393470764,
      "logits/rejected": -0.3221715986728668,
      "logps/chosen": -3.515720844268799,
      "logps/rejected": -4.333282470703125,
      "loss": 0.6929,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0163725558668375,
      "rewards/margins": 0.029236530885100365,
      "rewards/rejected": -0.04560908302664757,
      "step": 110
    },
    {
      "epoch": 0.09205983889528194,
      "grad_norm": 1.4560517072677612,
      "learning_rate": 4.5395241749808135e-05,
      "logits/chosen": -0.48749834299087524,
      "logits/rejected": -0.4916568696498871,
      "logps/chosen": -3.262430191040039,
      "logps/rejected": -3.4948978424072266,
      "loss": 0.6837,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.01899554952979088,
      "rewards/margins": 0.03163793310523033,
      "rewards/rejected": -0.05063347890973091,
      "step": 120
    },
    {
      "epoch": 0.09973149213655542,
      "grad_norm": 1.2213283777236938,
      "learning_rate": 4.501151189562548e-05,
      "logits/chosen": -0.4065719544887543,
      "logits/rejected": -0.4011871814727783,
      "logps/chosen": -3.5878148078918457,
      "logps/rejected": -4.147099494934082,
      "loss": 0.703,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.047407474368810654,
      "rewards/margins": 0.005889671854674816,
      "rewards/rejected": -0.05329713970422745,
      "step": 130
    },
    {
      "epoch": 0.10740314537782893,
      "grad_norm": 2.1541309356689453,
      "learning_rate": 4.4627782041442826e-05,
      "logits/chosen": -0.48593592643737793,
      "logits/rejected": -0.4823059141635895,
      "logps/chosen": -3.6085586547851562,
      "logps/rejected": -3.899576187133789,
      "loss": 0.7012,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06346064805984497,
      "rewards/margins": -0.018461909145116806,
      "rewards/rejected": -0.044998738914728165,
      "step": 140
    },
    {
      "epoch": 0.11507479861910241,
      "grad_norm": 1.4447782039642334,
      "learning_rate": 4.424405218726017e-05,
      "logits/chosen": -0.5145712494850159,
      "logits/rejected": -0.5148640275001526,
      "logps/chosen": -4.518651962280273,
      "logps/rejected": -3.0175552368164062,
      "loss": 0.6934,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.06713158637285233,
      "rewards/margins": -0.025863245129585266,
      "rewards/rejected": -0.04126833751797676,
      "step": 150
    },
    {
      "epoch": 0.11507479861910241,
      "eval_logits/chosen": -0.2573604881763458,
      "eval_logits/rejected": -0.25465065240859985,
      "eval_logps/chosen": -3.439135789871216,
      "eval_logps/rejected": -3.6858487129211426,
      "eval_loss": 0.691031813621521,
      "eval_rewards/accuracies": 0.512499988079071,
      "eval_rewards/chosen": -0.04062613099813461,
      "eval_rewards/margins": -0.005622314754873514,
      "eval_rewards/rejected": -0.035003818571567535,
      "eval_runtime": 87.701,
      "eval_samples_per_second": 29.122,
      "eval_steps_per_second": 0.912,
      "step": 150
    },
    {
      "epoch": 0.12274645186037592,
      "grad_norm": 2.1208548545837402,
      "learning_rate": 4.386032233307752e-05,
      "logits/chosen": -0.4267118573188782,
      "logits/rejected": -0.42528724670410156,
      "logps/chosen": -2.49307918548584,
      "logps/rejected": -5.034300804138184,
      "loss": 0.6927,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.02940777875483036,
      "rewards/margins": 0.01965685375034809,
      "rewards/rejected": -0.04906462877988815,
      "step": 160
    },
    {
      "epoch": 0.13041810510164942,
      "grad_norm": 1.4776760339736938,
      "learning_rate": 4.347659247889486e-05,
      "logits/chosen": -0.5030231475830078,
      "logits/rejected": -0.5042341947555542,
      "logps/chosen": -2.0654220581054688,
      "logps/rejected": -4.4005961418151855,
      "loss": 0.6875,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": 0.0012591822305694222,
      "rewards/margins": 0.07784555107355118,
      "rewards/rejected": -0.07658636569976807,
      "step": 170
    },
    {
      "epoch": 0.1380897583429229,
      "grad_norm": 1.282079815864563,
      "learning_rate": 4.309286262471221e-05,
      "logits/chosen": -0.45819562673568726,
      "logits/rejected": -0.4634513258934021,
      "logps/chosen": -3.5271224975585938,
      "logps/rejected": -3.9698429107666016,
      "loss": 0.6878,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.026808280497789383,
      "rewards/margins": 0.02008042298257351,
      "rewards/rejected": -0.04688870906829834,
      "step": 180
    },
    {
      "epoch": 0.1457614115841964,
      "grad_norm": 1.368600606918335,
      "learning_rate": 4.270913277052955e-05,
      "logits/chosen": -0.384149432182312,
      "logits/rejected": -0.3903164267539978,
      "logps/chosen": -3.522733211517334,
      "logps/rejected": -3.272055149078369,
      "loss": 0.7013,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.037452831864356995,
      "rewards/margins": -0.00283648818731308,
      "rewards/rejected": -0.034616343677043915,
      "step": 190
    },
    {
      "epoch": 0.1534330648254699,
      "grad_norm": 1.9209519624710083,
      "learning_rate": 4.23254029163469e-05,
      "logits/chosen": -0.5128501653671265,
      "logits/rejected": -0.5161421298980713,
      "logps/chosen": -2.724151134490967,
      "logps/rejected": -3.3391120433807373,
      "loss": 0.6932,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.008422130718827248,
      "rewards/margins": 0.02306986413896084,
      "rewards/rejected": -0.03149199113249779,
      "step": 200
    },
    {
      "epoch": 0.1534330648254699,
      "eval_logits/chosen": -0.2552909255027771,
      "eval_logits/rejected": -0.25272154808044434,
      "eval_logps/chosen": -3.351407527923584,
      "eval_logps/rejected": -3.6925933361053467,
      "eval_loss": 0.6919052600860596,
      "eval_rewards/accuracies": 0.528124988079071,
      "eval_rewards/chosen": -0.031853340566158295,
      "eval_rewards/margins": 0.0038249720819294453,
      "eval_rewards/rejected": -0.03567831218242645,
      "eval_runtime": 87.7075,
      "eval_samples_per_second": 29.12,
      "eval_steps_per_second": 0.912,
      "step": 200
    },
    {
      "epoch": 0.1611047180667434,
      "grad_norm": 1.575499415397644,
      "learning_rate": 4.194167306216424e-05,
      "logits/chosen": -0.45738959312438965,
      "logits/rejected": -0.45437225699424744,
      "logps/chosen": -4.087313175201416,
      "logps/rejected": -2.974914073944092,
      "loss": 0.7002,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.051591504365205765,
      "rewards/margins": -0.020506689324975014,
      "rewards/rejected": -0.0310848206281662,
      "step": 210
    },
    {
      "epoch": 0.16877637130801687,
      "grad_norm": 1.5426521301269531,
      "learning_rate": 4.155794320798158e-05,
      "logits/chosen": -0.4632241725921631,
      "logits/rejected": -0.4625597596168518,
      "logps/chosen": -2.200758457183838,
      "logps/rejected": -3.836440324783325,
      "loss": 0.6852,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.030624931678175926,
      "rewards/margins": 0.11931898444890976,
      "rewards/rejected": -0.08869405090808868,
      "step": 220
    },
    {
      "epoch": 0.17644802454929037,
      "grad_norm": 1.3270576000213623,
      "learning_rate": 4.117421335379893e-05,
      "logits/chosen": -0.4626352787017822,
      "logits/rejected": -0.4678708016872406,
      "logps/chosen": -2.704134464263916,
      "logps/rejected": -3.5658984184265137,
      "loss": 0.6898,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.010677630081772804,
      "rewards/margins": 0.02222255989909172,
      "rewards/rejected": -0.032900188118219376,
      "step": 230
    },
    {
      "epoch": 0.18411967779056387,
      "grad_norm": 1.4585351943969727,
      "learning_rate": 4.0790483499616274e-05,
      "logits/chosen": -0.32787808775901794,
      "logits/rejected": -0.3283527195453644,
      "logps/chosen": -3.4351840019226074,
      "logps/rejected": -3.090237617492676,
      "loss": 0.6953,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.02141396515071392,
      "rewards/margins": -0.00582413375377655,
      "rewards/rejected": -0.01558983325958252,
      "step": 240
    },
    {
      "epoch": 0.19179133103183735,
      "grad_norm": 1.5329344272613525,
      "learning_rate": 4.0406753645433616e-05,
      "logits/chosen": -0.4365290105342865,
      "logits/rejected": -0.431086927652359,
      "logps/chosen": -4.269787788391113,
      "logps/rejected": -2.5051932334899902,
      "loss": 0.7021,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.03114442527294159,
      "rewards/margins": -0.03054250404238701,
      "rewards/rejected": -0.0006019207648932934,
      "step": 250
    },
    {
      "epoch": 0.19179133103183735,
      "eval_logits/chosen": -0.25592881441116333,
      "eval_logits/rejected": -0.2537047863006592,
      "eval_logps/chosen": -2.992680311203003,
      "eval_logps/rejected": -3.351801633834839,
      "eval_loss": 0.6899347901344299,
      "eval_rewards/accuracies": 0.5249999761581421,
      "eval_rewards/chosen": 0.00401937635615468,
      "eval_rewards/margins": 0.005618534050881863,
      "eval_rewards/rejected": -0.0015991575783118606,
      "eval_runtime": 87.6512,
      "eval_samples_per_second": 29.138,
      "eval_steps_per_second": 0.913,
      "step": 250
    },
    {
      "epoch": 0.19946298427311085,
      "grad_norm": 2.1084399223327637,
      "learning_rate": 4.0023023791250965e-05,
      "logits/chosen": -0.4519824981689453,
      "logits/rejected": -0.4398951530456543,
      "logps/chosen": -2.484307050704956,
      "logps/rejected": -3.2058215141296387,
      "loss": 0.6891,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.010583693161606789,
      "rewards/margins": 0.03147741034626961,
      "rewards/rejected": -0.02089371532201767,
      "step": 260
    },
    {
      "epoch": 0.20713463751438435,
      "grad_norm": 1.2669779062271118,
      "learning_rate": 3.963929393706831e-05,
      "logits/chosen": -0.4561208188533783,
      "logits/rejected": -0.46242856979370117,
      "logps/chosen": -2.3247485160827637,
      "logps/rejected": -4.493135929107666,
      "loss": 0.6886,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.0006832152721472085,
      "rewards/margins": 0.01231946051120758,
      "rewards/rejected": -0.013002678751945496,
      "step": 270
    },
    {
      "epoch": 0.21480629075565785,
      "grad_norm": 2.359649181365967,
      "learning_rate": 3.925556408288565e-05,
      "logits/chosen": -0.41936391592025757,
      "logits/rejected": -0.4227515161037445,
      "logps/chosen": -3.846536636352539,
      "logps/rejected": -2.6847872734069824,
      "loss": 0.6942,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0359012708067894,
      "rewards/margins": -0.027397548779845238,
      "rewards/rejected": -0.00850372388958931,
      "step": 280
    },
    {
      "epoch": 0.22247794399693133,
      "grad_norm": 2.732644557952881,
      "learning_rate": 3.8871834228703e-05,
      "logits/chosen": -0.41848206520080566,
      "logits/rejected": -0.42105674743652344,
      "logps/chosen": -3.4439868927001953,
      "logps/rejected": -3.4636013507843018,
      "loss": 0.692,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.038081876933574677,
      "rewards/margins": -0.0026010065339505672,
      "rewards/rejected": -0.03548087924718857,
      "step": 290
    },
    {
      "epoch": 0.23014959723820483,
      "grad_norm": 1.7410922050476074,
      "learning_rate": 3.848810437452034e-05,
      "logits/chosen": -0.41233521699905396,
      "logits/rejected": -0.4056723117828369,
      "logps/chosen": -2.7670276165008545,
      "logps/rejected": -4.538814544677734,
      "loss": 0.6859,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.03271554037928581,
      "rewards/margins": 0.008648352697491646,
      "rewards/rejected": -0.04136389121413231,
      "step": 300
    },
    {
      "epoch": 0.23014959723820483,
      "eval_logits/chosen": -0.25867313146591187,
      "eval_logits/rejected": -0.2568739354610443,
      "eval_logps/chosen": -3.2079968452453613,
      "eval_logps/rejected": -3.6974151134490967,
      "eval_loss": 0.6905180215835571,
      "eval_rewards/accuracies": 0.543749988079071,
      "eval_rewards/chosen": -0.01751227304339409,
      "eval_rewards/margins": 0.018648196011781693,
      "eval_rewards/rejected": -0.03616046532988548,
      "eval_runtime": 87.8864,
      "eval_samples_per_second": 29.06,
      "eval_steps_per_second": 0.91,
      "step": 300
    },
    {
      "epoch": 0.23782125047947833,
      "grad_norm": 1.6948188543319702,
      "learning_rate": 3.810437452033768e-05,
      "logits/chosen": -0.4763808250427246,
      "logits/rejected": -0.47943514585494995,
      "logps/chosen": -3.1502089500427246,
      "logps/rejected": -4.5027756690979,
      "loss": 0.6943,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.018290940672159195,
      "rewards/margins": 0.01174425520002842,
      "rewards/rejected": -0.030035194009542465,
      "step": 310
    },
    {
      "epoch": 0.24549290372075183,
      "grad_norm": 1.2756513357162476,
      "learning_rate": 3.772064466615503e-05,
      "logits/chosen": -0.5276287198066711,
      "logits/rejected": -0.5248805284500122,
      "logps/chosen": -3.315579891204834,
      "logps/rejected": -2.8239669799804688,
      "loss": 0.689,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": 0.0013980341609567404,
      "rewards/margins": 0.03776642307639122,
      "rewards/rejected": -0.036368388682603836,
      "step": 320
    },
    {
      "epoch": 0.25316455696202533,
      "grad_norm": 1.7512315511703491,
      "learning_rate": 3.733691481197237e-05,
      "logits/chosen": -0.4925498366355896,
      "logits/rejected": -0.49201974272727966,
      "logps/chosen": -2.7408628463745117,
      "logps/rejected": -3.967554807662964,
      "loss": 0.6947,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.011951248161494732,
      "rewards/margins": 0.022660424932837486,
      "rewards/rejected": -0.01070917397737503,
      "step": 330
    },
    {
      "epoch": 0.26083621020329883,
      "grad_norm": 2.073042869567871,
      "learning_rate": 3.6953184957789715e-05,
      "logits/chosen": -0.4371699392795563,
      "logits/rejected": -0.4345782697200775,
      "logps/chosen": -2.061657190322876,
      "logps/rejected": -3.6886394023895264,
      "loss": 0.691,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": 0.018956389278173447,
      "rewards/margins": 0.023976778611540794,
      "rewards/rejected": -0.005020388402044773,
      "step": 340
    },
    {
      "epoch": 0.2685078634445723,
      "grad_norm": 1.3207100629806519,
      "learning_rate": 3.6569455103607064e-05,
      "logits/chosen": -0.3427134156227112,
      "logits/rejected": -0.3307674825191498,
      "logps/chosen": -2.7634048461914062,
      "logps/rejected": -3.556063413619995,
      "loss": 0.6961,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.006752232555299997,
      "rewards/margins": 0.004960778169333935,
      "rewards/rejected": -0.01171301119029522,
      "step": 350
    },
    {
      "epoch": 0.2685078634445723,
      "eval_logits/chosen": -0.2583015561103821,
      "eval_logits/rejected": -0.2559833824634552,
      "eval_logps/chosen": -3.264441728591919,
      "eval_logps/rejected": -3.69817852973938,
      "eval_loss": 0.6912429928779602,
      "eval_rewards/accuracies": 0.5375000238418579,
      "eval_rewards/chosen": -0.023156724870204926,
      "eval_rewards/margins": 0.013080140575766563,
      "eval_rewards/rejected": -0.03623687103390694,
      "eval_runtime": 87.903,
      "eval_samples_per_second": 29.055,
      "eval_steps_per_second": 0.91,
      "step": 350
    },
    {
      "epoch": 0.2761795166858458,
      "grad_norm": 1.3935655355453491,
      "learning_rate": 3.6185725249424406e-05,
      "logits/chosen": -0.34501805901527405,
      "logits/rejected": -0.3470131456851959,
      "logps/chosen": -3.49810791015625,
      "logps/rejected": -3.8505330085754395,
      "loss": 0.689,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.05252550169825554,
      "rewards/margins": -0.009586334228515625,
      "rewards/rejected": -0.042939163744449615,
      "step": 360
    },
    {
      "epoch": 0.2838511699271193,
      "grad_norm": 1.6383577585220337,
      "learning_rate": 3.580199539524175e-05,
      "logits/chosen": -0.48913854360580444,
      "logits/rejected": -0.4845835566520691,
      "logps/chosen": -3.775874376296997,
      "logps/rejected": -2.7326130867004395,
      "loss": 0.6899,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04597536846995354,
      "rewards/margins": -0.018767457455396652,
      "rewards/rejected": -0.027207905426621437,
      "step": 370
    },
    {
      "epoch": 0.2915228231683928,
      "grad_norm": 1.5284889936447144,
      "learning_rate": 3.54182655410591e-05,
      "logits/chosen": -0.3916495442390442,
      "logits/rejected": -0.3948409855365753,
      "logps/chosen": -3.680704116821289,
      "logps/rejected": -3.731623411178589,
      "loss": 0.6865,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.02748117782175541,
      "rewards/margins": 0.013354331254959106,
      "rewards/rejected": -0.04083551466464996,
      "step": 380
    },
    {
      "epoch": 0.2991944764096663,
      "grad_norm": 1.75348699092865,
      "learning_rate": 3.503453568687644e-05,
      "logits/chosen": -0.3796854615211487,
      "logits/rejected": -0.38789474964141846,
      "logps/chosen": -2.4654390811920166,
      "logps/rejected": -5.180593490600586,
      "loss": 0.6917,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.019467497244477272,
      "rewards/margins": 0.08367259800434113,
      "rewards/rejected": -0.10314009338617325,
      "step": 390
    },
    {
      "epoch": 0.3068661296509398,
      "grad_norm": 1.9211453199386597,
      "learning_rate": 3.465080583269378e-05,
      "logits/chosen": -0.42692264914512634,
      "logits/rejected": -0.42641615867614746,
      "logps/chosen": -2.586097240447998,
      "logps/rejected": -4.711082458496094,
      "loss": 0.6978,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.04758203774690628,
      "rewards/margins": 0.018220845609903336,
      "rewards/rejected": -0.06580288708209991,
      "step": 400
    },
    {
      "epoch": 0.3068661296509398,
      "eval_logits/chosen": -0.25191357731819153,
      "eval_logits/rejected": -0.24964801967144012,
      "eval_logps/chosen": -3.459468126296997,
      "eval_logps/rejected": -3.9026706218719482,
      "eval_loss": 0.6893244385719299,
      "eval_rewards/accuracies": 0.5249999761581421,
      "eval_rewards/chosen": -0.04265937954187393,
      "eval_rewards/margins": 0.014026666060090065,
      "eval_rewards/rejected": -0.05668604373931885,
      "eval_runtime": 87.8645,
      "eval_samples_per_second": 29.068,
      "eval_steps_per_second": 0.91,
      "step": 400
    },
    {
      "epoch": 0.3145377828922133,
      "grad_norm": 1.5234233140945435,
      "learning_rate": 3.426707597851113e-05,
      "logits/chosen": -0.28339290618896484,
      "logits/rejected": -0.27566584944725037,
      "logps/chosen": -3.7383079528808594,
      "logps/rejected": -3.349996566772461,
      "loss": 0.6824,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.04696209728717804,
      "rewards/margins": -0.0003062840551137924,
      "rewards/rejected": -0.0466558113694191,
      "step": 410
    },
    {
      "epoch": 0.3222094361334868,
      "grad_norm": 1.5943760871887207,
      "learning_rate": 3.388334612432847e-05,
      "logits/chosen": -0.440364271402359,
      "logits/rejected": -0.4411134719848633,
      "logps/chosen": -2.43839430809021,
      "logps/rejected": -4.264366626739502,
      "loss": 0.6874,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.002678838325664401,
      "rewards/margins": 0.06886979192495346,
      "rewards/rejected": -0.07154864072799683,
      "step": 420
    },
    {
      "epoch": 0.32988108937476024,
      "grad_norm": 2.3275911808013916,
      "learning_rate": 3.3499616270145814e-05,
      "logits/chosen": -0.4382743239402771,
      "logits/rejected": -0.4452611804008484,
      "logps/chosen": -4.034234523773193,
      "logps/rejected": -3.718547821044922,
      "loss": 0.6962,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05849426984786987,
      "rewards/margins": 0.03669988363981247,
      "rewards/rejected": -0.09519414603710175,
      "step": 430
    },
    {
      "epoch": 0.33755274261603374,
      "grad_norm": 1.7006807327270508,
      "learning_rate": 3.311588641596316e-05,
      "logits/chosen": -0.488353431224823,
      "logits/rejected": -0.4902741014957428,
      "logps/chosen": -2.8092918395996094,
      "logps/rejected": -4.1550493240356445,
      "loss": 0.6961,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.017056407406926155,
      "rewards/margins": 0.026486407965421677,
      "rewards/rejected": -0.04354281350970268,
      "step": 440
    },
    {
      "epoch": 0.34522439585730724,
      "grad_norm": 2.1540279388427734,
      "learning_rate": 3.2732156561780505e-05,
      "logits/chosen": -0.46933096647262573,
      "logits/rejected": -0.47831687331199646,
      "logps/chosen": -3.4161593914031982,
      "logps/rejected": -3.7839865684509277,
      "loss": 0.6963,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.062318772077560425,
      "rewards/margins": -0.014327460899949074,
      "rewards/rejected": -0.0479913167655468,
      "step": 450
    },
    {
      "epoch": 0.34522439585730724,
      "eval_logits/chosen": -0.2543574869632721,
      "eval_logits/rejected": -0.25201791524887085,
      "eval_logps/chosen": -3.321566343307495,
      "eval_logps/rejected": -3.75457763671875,
      "eval_loss": 0.6880484819412231,
      "eval_rewards/accuracies": 0.5249999761581421,
      "eval_rewards/chosen": -0.028869222849607468,
      "eval_rewards/margins": 0.01300754863768816,
      "eval_rewards/rejected": -0.04187677055597305,
      "eval_runtime": 87.6848,
      "eval_samples_per_second": 29.127,
      "eval_steps_per_second": 0.912,
      "step": 450
    },
    {
      "epoch": 0.35289604909858074,
      "grad_norm": 2.41361665725708,
      "learning_rate": 3.2348426707597854e-05,
      "logits/chosen": -0.517076849937439,
      "logits/rejected": -0.5212947130203247,
      "logps/chosen": -3.707463026046753,
      "logps/rejected": -2.6402366161346436,
      "loss": 0.6878,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.03469429910182953,
      "rewards/margins": -0.0030276328325271606,
      "rewards/rejected": -0.03166666626930237,
      "step": 460
    },
    {
      "epoch": 0.36056770233985425,
      "grad_norm": 1.4909201860427856,
      "learning_rate": 3.1964696853415196e-05,
      "logits/chosen": -0.44358164072036743,
      "logits/rejected": -0.4401718080043793,
      "logps/chosen": -2.663454532623291,
      "logps/rejected": -4.2893218994140625,
      "loss": 0.6871,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.005995375569909811,
      "rewards/margins": 0.09306643158197403,
      "rewards/rejected": -0.08707105368375778,
      "step": 470
    },
    {
      "epoch": 0.36823935558112775,
      "grad_norm": 1.5664546489715576,
      "learning_rate": 3.1580966999232545e-05,
      "logits/chosen": -0.5029574632644653,
      "logits/rejected": -0.5032514333724976,
      "logps/chosen": -3.0805327892303467,
      "logps/rejected": -3.4728941917419434,
      "loss": 0.7022,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03778943419456482,
      "rewards/margins": 0.011032382026314735,
      "rewards/rejected": -0.048821814358234406,
      "step": 480
    },
    {
      "epoch": 0.37591100882240125,
      "grad_norm": 1.4422327280044556,
      "learning_rate": 3.119723714504989e-05,
      "logits/chosen": -0.4989328384399414,
      "logits/rejected": -0.49760347604751587,
      "logps/chosen": -3.063786745071411,
      "logps/rejected": -3.134028434753418,
      "loss": 0.6919,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03893281891942024,
      "rewards/margins": -0.007891304790973663,
      "rewards/rejected": -0.03104151412844658,
      "step": 490
    },
    {
      "epoch": 0.3835826620636747,
      "grad_norm": 1.7234938144683838,
      "learning_rate": 3.0813507290867236e-05,
      "logits/chosen": -0.4708559513092041,
      "logits/rejected": -0.4755636155605316,
      "logps/chosen": -3.4675452709198,
      "logps/rejected": -4.40737247467041,
      "loss": 0.6909,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.02458677999675274,
      "rewards/margins": 0.0017846006667241454,
      "rewards/rejected": -0.02637138031423092,
      "step": 500
    },
    {
      "epoch": 0.3835826620636747,
      "eval_logits/chosen": -0.25855669379234314,
      "eval_logits/rejected": -0.2564457058906555,
      "eval_logps/chosen": -3.37125825881958,
      "eval_logps/rejected": -3.7394096851348877,
      "eval_loss": 0.6893431544303894,
      "eval_rewards/accuracies": 0.515625,
      "eval_rewards/chosen": -0.03383838012814522,
      "eval_rewards/margins": 0.006521561183035374,
      "eval_rewards/rejected": -0.04035994037985802,
      "eval_runtime": 87.8159,
      "eval_samples_per_second": 29.084,
      "eval_steps_per_second": 0.911,
      "step": 500
    },
    {
      "epoch": 0.3912543153049482,
      "grad_norm": 1.5501247644424438,
      "learning_rate": 3.0429777436684575e-05,
      "logits/chosen": -0.4411280155181885,
      "logits/rejected": -0.4376070499420166,
      "logps/chosen": -3.4258601665496826,
      "logps/rejected": -3.10990309715271,
      "loss": 0.6846,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0510549321770668,
      "rewards/margins": -0.015095305629074574,
      "rewards/rejected": -0.035959623754024506,
      "step": 510
    },
    {
      "epoch": 0.3989259685462217,
      "grad_norm": 1.5012112855911255,
      "learning_rate": 3.0046047582501917e-05,
      "logits/chosen": -0.4509127140045166,
      "logits/rejected": -0.4517589509487152,
      "logps/chosen": -2.4196524620056152,
      "logps/rejected": -3.98419451713562,
      "loss": 0.6872,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0005872026085853577,
      "rewards/margins": 0.0763540267944336,
      "rewards/rejected": -0.07576681673526764,
      "step": 520
    },
    {
      "epoch": 0.4065976217874952,
      "grad_norm": 2.1596052646636963,
      "learning_rate": 2.9662317728319266e-05,
      "logits/chosen": -0.41014641523361206,
      "logits/rejected": -0.4043218195438385,
      "logps/chosen": -3.8312087059020996,
      "logps/rejected": -2.908543109893799,
      "loss": 0.6886,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.02407076209783554,
      "rewards/margins": 0.02513458952307701,
      "rewards/rejected": -0.04920535162091255,
      "step": 530
    },
    {
      "epoch": 0.4142692750287687,
      "grad_norm": 1.4137400388717651,
      "learning_rate": 2.9278587874136608e-05,
      "logits/chosen": -0.4017483592033386,
      "logits/rejected": -0.39557427167892456,
      "logps/chosen": -3.8227367401123047,
      "logps/rejected": -3.195641279220581,
      "loss": 0.6934,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.03194689378142357,
      "rewards/margins": -0.01945153810083866,
      "rewards/rejected": -0.012495351955294609,
      "step": 540
    },
    {
      "epoch": 0.4219409282700422,
      "grad_norm": 1.4084391593933105,
      "learning_rate": 2.8894858019953953e-05,
      "logits/chosen": -0.4596850872039795,
      "logits/rejected": -0.4715039134025574,
      "logps/chosen": -3.1471800804138184,
      "logps/rejected": -3.90801739692688,
      "loss": 0.6944,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.019990995526313782,
      "rewards/margins": 0.018570415675640106,
      "rewards/rejected": -0.03856140747666359,
      "step": 550
    },
    {
      "epoch": 0.4219409282700422,
      "eval_logits/chosen": -0.26123708486557007,
      "eval_logits/rejected": -0.25891199707984924,
      "eval_logps/chosen": -3.1744627952575684,
      "eval_logps/rejected": -3.591878890991211,
      "eval_loss": 0.6874387264251709,
      "eval_rewards/accuracies": 0.515625,
      "eval_rewards/chosen": -0.014158882200717926,
      "eval_rewards/margins": 0.011447983793914318,
      "eval_rewards/rejected": -0.02560686506330967,
      "eval_runtime": 90.5755,
      "eval_samples_per_second": 28.197,
      "eval_steps_per_second": 0.883,
      "step": 550
    },
    {
      "epoch": 0.4296125815113157,
      "grad_norm": 3.040137529373169,
      "learning_rate": 2.85111281657713e-05,
      "logits/chosen": -0.41563597321510315,
      "logits/rejected": -0.41134652495384216,
      "logps/chosen": -2.954399824142456,
      "logps/rejected": -2.869027853012085,
      "loss": 0.7005,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.04915647208690643,
      "rewards/margins": 0.0781073272228241,
      "rewards/rejected": -0.02895086072385311,
      "step": 560
    },
    {
      "epoch": 0.4372842347525892,
      "grad_norm": 1.730731725692749,
      "learning_rate": 2.8127398311588644e-05,
      "logits/chosen": -0.43522629141807556,
      "logits/rejected": -0.42925986647605896,
      "logps/chosen": -3.231485366821289,
      "logps/rejected": -3.533066511154175,
      "loss": 0.6926,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": 0.012996886856853962,
      "rewards/margins": 0.05765851214528084,
      "rewards/rejected": -0.04466162249445915,
      "step": 570
    },
    {
      "epoch": 0.44495588799386265,
      "grad_norm": 1.4563325643539429,
      "learning_rate": 2.7743668457405987e-05,
      "logits/chosen": -0.4720652103424072,
      "logits/rejected": -0.46499529480934143,
      "logps/chosen": -3.880791187286377,
      "logps/rejected": -3.5834224224090576,
      "loss": 0.7003,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.055101800709962845,
      "rewards/margins": -0.01127576269209385,
      "rewards/rejected": -0.043826036155223846,
      "step": 580
    },
    {
      "epoch": 0.45262754123513615,
      "grad_norm": 1.680908441543579,
      "learning_rate": 2.7359938603223335e-05,
      "logits/chosen": -0.2684255540370941,
      "logits/rejected": -0.2682047188282013,
      "logps/chosen": -2.8263630867004395,
      "logps/rejected": -3.5271003246307373,
      "loss": 0.6887,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": 0.018410243093967438,
      "rewards/margins": 0.04611648991703987,
      "rewards/rejected": -0.027706250548362732,
      "step": 590
    },
    {
      "epoch": 0.46029919447640966,
      "grad_norm": 1.4875671863555908,
      "learning_rate": 2.6976208749040678e-05,
      "logits/chosen": -0.5006714463233948,
      "logits/rejected": -0.5000745058059692,
      "logps/chosen": -2.8757147789001465,
      "logps/rejected": -3.4219863414764404,
      "loss": 0.6837,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.022861991077661514,
      "rewards/margins": 0.0631416067481041,
      "rewards/rejected": -0.04027962684631348,
      "step": 600
    },
    {
      "epoch": 0.46029919447640966,
      "eval_logits/chosen": -0.25836092233657837,
      "eval_logits/rejected": -0.2553294599056244,
      "eval_logps/chosen": -3.034031867980957,
      "eval_logps/rejected": -3.487445116043091,
      "eval_loss": 0.6917591691017151,
      "eval_rewards/accuracies": 0.515625,
      "eval_rewards/chosen": -0.00011577569239307195,
      "eval_rewards/margins": 0.015047714114189148,
      "eval_rewards/rejected": -0.015163486823439598,
      "eval_runtime": 95.32,
      "eval_samples_per_second": 26.794,
      "eval_steps_per_second": 0.839,
      "step": 600
    },
    {
      "epoch": 0.46797084771768316,
      "grad_norm": 1.0080013275146484,
      "learning_rate": 2.659247889485802e-05,
      "logits/chosen": -0.4055057466030121,
      "logits/rejected": -0.4117734432220459,
      "logps/chosen": -2.3307080268859863,
      "logps/rejected": -3.9688384532928467,
      "loss": 0.6911,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.004725713282823563,
      "rewards/margins": -4.6133995965647046e-06,
      "rewards/rejected": -0.004721099976450205,
      "step": 610
    },
    {
      "epoch": 0.47564250095895666,
      "grad_norm": 1.2446097135543823,
      "learning_rate": 2.620874904067537e-05,
      "logits/chosen": -0.3564910292625427,
      "logits/rejected": -0.3598317801952362,
      "logps/chosen": -2.8816721439361572,
      "logps/rejected": -3.8162410259246826,
      "loss": 0.7045,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.04238850623369217,
      "rewards/margins": -0.050994087010622025,
      "rewards/rejected": 0.00860558170825243,
      "step": 620
    },
    {
      "epoch": 0.48331415420023016,
      "grad_norm": 2.1006405353546143,
      "learning_rate": 2.582501918649271e-05,
      "logits/chosen": -0.4763752520084381,
      "logits/rejected": -0.4724522531032562,
      "logps/chosen": -3.6634361743927,
      "logps/rejected": -3.523616313934326,
      "loss": 0.6876,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.015630219131708145,
      "rewards/margins": 0.03971021622419357,
      "rewards/rejected": -0.05534043163061142,
      "step": 630
    },
    {
      "epoch": 0.49098580744150366,
      "grad_norm": 1.9049338102340698,
      "learning_rate": 2.5441289332310053e-05,
      "logits/chosen": -0.46114760637283325,
      "logits/rejected": -0.4514070451259613,
      "logps/chosen": -3.082925796508789,
      "logps/rejected": -3.305220365524292,
      "loss": 0.6923,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.03719641640782356,
      "rewards/margins": -0.010646897368133068,
      "rewards/rejected": -0.02654952183365822,
      "step": 640
    },
    {
      "epoch": 0.49865746068277716,
      "grad_norm": 1.5372164249420166,
      "learning_rate": 2.50575594781274e-05,
      "logits/chosen": -0.4374101758003235,
      "logits/rejected": -0.4314614236354828,
      "logps/chosen": -3.3405063152313232,
      "logps/rejected": -3.743143081665039,
      "loss": 0.6894,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.011367046274244785,
      "rewards/margins": 0.05027226731181145,
      "rewards/rejected": -0.061639316380023956,
      "step": 650
    },
    {
      "epoch": 0.49865746068277716,
      "eval_logits/chosen": -0.25022387504577637,
      "eval_logits/rejected": -0.24710026383399963,
      "eval_logps/chosen": -3.5461456775665283,
      "eval_logps/rejected": -3.8798840045928955,
      "eval_loss": 0.688687801361084,
      "eval_rewards/accuracies": 0.49687498807907104,
      "eval_rewards/chosen": -0.0513271763920784,
      "eval_rewards/margins": 0.0030801701359450817,
      "eval_rewards/rejected": -0.05440734699368477,
      "eval_runtime": 89.1213,
      "eval_samples_per_second": 28.658,
      "eval_steps_per_second": 0.898,
      "step": 650
    },
    {
      "epoch": 0.5063291139240507,
      "grad_norm": 1.4859964847564697,
      "learning_rate": 2.4673829623944744e-05,
      "logits/chosen": -0.4470909535884857,
      "logits/rejected": -0.4404575228691101,
      "logps/chosen": -3.57057523727417,
      "logps/rejected": -4.062138080596924,
      "loss": 0.6839,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05643466114997864,
      "rewards/margins": 0.012388080358505249,
      "rewards/rejected": -0.06882274150848389,
      "step": 660
    },
    {
      "epoch": 0.5140007671653242,
      "grad_norm": 2.348560333251953,
      "learning_rate": 2.429009976976209e-05,
      "logits/chosen": -0.4324890971183777,
      "logits/rejected": -0.4288514256477356,
      "logps/chosen": -3.4150989055633545,
      "logps/rejected": -3.761260509490967,
      "loss": 0.6905,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.047535546123981476,
      "rewards/margins": 0.00843095313757658,
      "rewards/rejected": -0.05596649646759033,
      "step": 670
    },
    {
      "epoch": 0.5216724204065977,
      "grad_norm": 2.114100933074951,
      "learning_rate": 2.390636991557943e-05,
      "logits/chosen": -0.4224484860897064,
      "logits/rejected": -0.41605424880981445,
      "logps/chosen": -3.8406059741973877,
      "logps/rejected": -3.227407455444336,
      "loss": 0.6923,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.09835998713970184,
      "rewards/margins": -0.028456950560212135,
      "rewards/rejected": -0.06990303844213486,
      "step": 680
    },
    {
      "epoch": 0.5293440736478712,
      "grad_norm": 2.143270254135132,
      "learning_rate": 2.3522640061396777e-05,
      "logits/chosen": -0.35925137996673584,
      "logits/rejected": -0.35858094692230225,
      "logps/chosen": -4.149369239807129,
      "logps/rejected": -4.207783222198486,
      "loss": 0.6901,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.07230309396982193,
      "rewards/margins": 0.022937068715691566,
      "rewards/rejected": -0.09524016082286835,
      "step": 690
    },
    {
      "epoch": 0.5370157268891446,
      "grad_norm": 1.3052890300750732,
      "learning_rate": 2.3138910207214122e-05,
      "logits/chosen": -0.4177284836769104,
      "logits/rejected": -0.4176989197731018,
      "logps/chosen": -2.371631622314453,
      "logps/rejected": -5.967281341552734,
      "loss": 0.6972,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.025050634518265724,
      "rewards/margins": 0.0979982241988182,
      "rewards/rejected": -0.12304886430501938,
      "step": 700
    },
    {
      "epoch": 0.5370157268891446,
      "eval_logits/chosen": -0.2624206840991974,
      "eval_logits/rejected": -0.259426087141037,
      "eval_logps/chosen": -3.501861572265625,
      "eval_logps/rejected": -3.8878912925720215,
      "eval_loss": 0.6871370673179626,
      "eval_rewards/accuracies": 0.48124998807907104,
      "eval_rewards/chosen": -0.046898726373910904,
      "eval_rewards/margins": 0.008309418335556984,
      "eval_rewards/rejected": -0.05520814657211304,
      "eval_runtime": 91.2498,
      "eval_samples_per_second": 27.989,
      "eval_steps_per_second": 0.877,
      "step": 700
    },
    {
      "epoch": 0.5446873801304181,
      "grad_norm": 1.2912267446517944,
      "learning_rate": 2.2755180353031468e-05,
      "logits/chosen": -0.45912307500839233,
      "logits/rejected": -0.4561159014701843,
      "logps/chosen": -3.2371792793273926,
      "logps/rejected": -3.8025176525115967,
      "loss": 0.6983,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.024336762726306915,
      "rewards/margins": 0.016023293137550354,
      "rewards/rejected": -0.040360044687986374,
      "step": 710
    },
    {
      "epoch": 0.5523590333716916,
      "grad_norm": 1.5830397605895996,
      "learning_rate": 2.2371450498848813e-05,
      "logits/chosen": -0.382514089345932,
      "logits/rejected": -0.3864609897136688,
      "logps/chosen": -1.6761538982391357,
      "logps/rejected": -4.973038673400879,
      "loss": 0.6851,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.012824362143874168,
      "rewards/margins": 0.0575835183262825,
      "rewards/rejected": -0.07040788233280182,
      "step": 720
    },
    {
      "epoch": 0.5600306866129651,
      "grad_norm": 1.851820707321167,
      "learning_rate": 2.198772064466616e-05,
      "logits/chosen": -0.30062246322631836,
      "logits/rejected": -0.3018190860748291,
      "logps/chosen": -2.90508770942688,
      "logps/rejected": -3.7549374103546143,
      "loss": 0.6959,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02952618896961212,
      "rewards/margins": 0.014450429007411003,
      "rewards/rejected": -0.04397661238908768,
      "step": 730
    },
    {
      "epoch": 0.5677023398542386,
      "grad_norm": 1.2601155042648315,
      "learning_rate": 2.16039907904835e-05,
      "logits/chosen": -0.5282914042472839,
      "logits/rejected": -0.5290226936340332,
      "logps/chosen": -2.747199535369873,
      "logps/rejected": -3.397148609161377,
      "loss": 0.6875,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": 0.023777935653924942,
      "rewards/margins": 0.060780882835388184,
      "rewards/rejected": -0.03700294345617294,
      "step": 740
    },
    {
      "epoch": 0.5753739930955121,
      "grad_norm": 1.9561665058135986,
      "learning_rate": 2.1220260936300846e-05,
      "logits/chosen": -0.46929964423179626,
      "logits/rejected": -0.4689921736717224,
      "logps/chosen": -3.4335849285125732,
      "logps/rejected": -3.539712429046631,
      "loss": 0.6861,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.01373897772282362,
      "rewards/margins": 0.052022386342287064,
      "rewards/rejected": -0.06576136499643326,
      "step": 750
    },
    {
      "epoch": 0.5753739930955121,
      "eval_logits/chosen": -0.26118698716163635,
      "eval_logits/rejected": -0.25837644934654236,
      "eval_logps/chosen": -3.3246681690216064,
      "eval_logps/rejected": -3.706329345703125,
      "eval_loss": 0.6877872943878174,
      "eval_rewards/accuracies": 0.512499988079071,
      "eval_rewards/chosen": -0.029179394245147705,
      "eval_rewards/margins": 0.007872538641095161,
      "eval_rewards/rejected": -0.03705193102359772,
      "eval_runtime": 95.7364,
      "eval_samples_per_second": 26.677,
      "eval_steps_per_second": 0.836,
      "step": 750
    },
    {
      "epoch": 0.5830456463367856,
      "grad_norm": 1.3404334783554077,
      "learning_rate": 2.0836531082118192e-05,
      "logits/chosen": -0.37314945459365845,
      "logits/rejected": -0.3766701817512512,
      "logps/chosen": -2.7383177280426025,
      "logps/rejected": -2.9454519748687744,
      "loss": 0.693,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.04017211124300957,
      "rewards/margins": -0.045864250510931015,
      "rewards/rejected": 0.005692136473953724,
      "step": 760
    },
    {
      "epoch": 0.5907172995780591,
      "grad_norm": 1.4485934972763062,
      "learning_rate": 2.0452801227935534e-05,
      "logits/chosen": -0.46926283836364746,
      "logits/rejected": -0.46992430090904236,
      "logps/chosen": -3.8600451946258545,
      "logps/rejected": -3.410430431365967,
      "loss": 0.6925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.031888850033283234,
      "rewards/margins": 0.010654223151504993,
      "rewards/rejected": -0.042543068528175354,
      "step": 770
    },
    {
      "epoch": 0.5983889528193326,
      "grad_norm": 1.8931002616882324,
      "learning_rate": 2.006907137375288e-05,
      "logits/chosen": -0.3966008424758911,
      "logits/rejected": -0.4044922888278961,
      "logps/chosen": -2.1048827171325684,
      "logps/rejected": -4.816119194030762,
      "loss": 0.6858,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.002983979182317853,
      "rewards/margins": 0.08909803628921509,
      "rewards/rejected": -0.0861140564084053,
      "step": 780
    },
    {
      "epoch": 0.6060606060606061,
      "grad_norm": 2.2583765983581543,
      "learning_rate": 1.9685341519570225e-05,
      "logits/chosen": -0.4961305558681488,
      "logits/rejected": -0.4885970652103424,
      "logps/chosen": -3.6672680377960205,
      "logps/rejected": -3.291787624359131,
      "loss": 0.692,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.02681976929306984,
      "rewards/margins": 0.014080104418098927,
      "rewards/rejected": -0.04089986905455589,
      "step": 790
    },
    {
      "epoch": 0.6137322593018796,
      "grad_norm": 1.8438897132873535,
      "learning_rate": 1.9301611665387567e-05,
      "logits/chosen": -0.46609607338905334,
      "logits/rejected": -0.4627665877342224,
      "logps/chosen": -3.5119385719299316,
      "logps/rejected": -3.595712184906006,
      "loss": 0.6866,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.025941628962755203,
      "rewards/margins": 0.030908849090337753,
      "rewards/rejected": -0.056850481778383255,
      "step": 800
    },
    {
      "epoch": 0.6137322593018796,
      "eval_logits/chosen": -0.2616404891014099,
      "eval_logits/rejected": -0.25945189595222473,
      "eval_logps/chosen": -3.2828381061553955,
      "eval_logps/rejected": -3.7320716381073,
      "eval_loss": 0.6877512335777283,
      "eval_rewards/accuracies": 0.518750011920929,
      "eval_rewards/chosen": -0.024996396154165268,
      "eval_rewards/margins": 0.014629754237830639,
      "eval_rewards/rejected": -0.03962615132331848,
      "eval_runtime": 96.9287,
      "eval_samples_per_second": 26.349,
      "eval_steps_per_second": 0.825,
      "step": 800
    },
    {
      "epoch": 0.6214039125431531,
      "grad_norm": 1.5462312698364258,
      "learning_rate": 1.8917881811204912e-05,
      "logits/chosen": -0.5312341451644897,
      "logits/rejected": -0.5319844484329224,
      "logps/chosen": -3.227830410003662,
      "logps/rejected": -3.9378840923309326,
      "loss": 0.6881,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.031698137521743774,
      "rewards/margins": 0.033506251871585846,
      "rewards/rejected": -0.06520438939332962,
      "step": 810
    },
    {
      "epoch": 0.6290755657844266,
      "grad_norm": 1.644169807434082,
      "learning_rate": 1.8534151957022258e-05,
      "logits/chosen": -0.42604535818099976,
      "logits/rejected": -0.42680415511131287,
      "logps/chosen": -3.025153875350952,
      "logps/rejected": -3.5833792686462402,
      "loss": 0.6951,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.017264336347579956,
      "rewards/margins": 0.03092106617987156,
      "rewards/rejected": -0.048185400664806366,
      "step": 820
    },
    {
      "epoch": 0.6367472190257001,
      "grad_norm": 2.567456007003784,
      "learning_rate": 1.81504221028396e-05,
      "logits/chosen": -0.3624061048030853,
      "logits/rejected": -0.3603518605232239,
      "logps/chosen": -2.727677583694458,
      "logps/rejected": -4.618671417236328,
      "loss": 0.6831,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": 0.013222327455878258,
      "rewards/margins": 0.04956119880080223,
      "rewards/rejected": -0.03633887320756912,
      "step": 830
    },
    {
      "epoch": 0.6444188722669736,
      "grad_norm": 1.6993279457092285,
      "learning_rate": 1.7766692248656946e-05,
      "logits/chosen": -0.43364423513412476,
      "logits/rejected": -0.43153029680252075,
      "logps/chosen": -3.7280707359313965,
      "logps/rejected": -3.3984951972961426,
      "loss": 0.6892,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.003762350883334875,
      "rewards/margins": 0.04601341485977173,
      "rewards/rejected": -0.04977576062083244,
      "step": 840
    },
    {
      "epoch": 0.652090525508247,
      "grad_norm": 1.2924683094024658,
      "learning_rate": 1.738296239447429e-05,
      "logits/chosen": -0.40035882592201233,
      "logits/rejected": -0.39904215931892395,
      "logps/chosen": -2.473344087600708,
      "logps/rejected": -4.267284393310547,
      "loss": 0.6901,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.02397814765572548,
      "rewards/margins": 0.036180444061756134,
      "rewards/rejected": -0.060158587992191315,
      "step": 850
    },
    {
      "epoch": 0.652090525508247,
      "eval_logits/chosen": -0.264896422624588,
      "eval_logits/rejected": -0.2623136341571808,
      "eval_logps/chosen": -3.455109119415283,
      "eval_logps/rejected": -3.909381866455078,
      "eval_loss": 0.6873416304588318,
      "eval_rewards/accuracies": 0.503125011920929,
      "eval_rewards/chosen": -0.04222346469759941,
      "eval_rewards/margins": 0.015133677050471306,
      "eval_rewards/rejected": -0.057357143610715866,
      "eval_runtime": 87.969,
      "eval_samples_per_second": 29.033,
      "eval_steps_per_second": 0.909,
      "step": 850
    },
    {
      "epoch": 0.6597621787495205,
      "grad_norm": 1.6212762594223022,
      "learning_rate": 1.6999232540291633e-05,
      "logits/chosen": -0.41735363006591797,
      "logits/rejected": -0.4102698266506195,
      "logps/chosen": -3.7062439918518066,
      "logps/rejected": -3.9006316661834717,
      "loss": 0.6915,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0529264435172081,
      "rewards/margins": 0.005926959216594696,
      "rewards/rejected": -0.0588533990085125,
      "step": 860
    },
    {
      "epoch": 0.667433831990794,
      "grad_norm": 2.2773942947387695,
      "learning_rate": 1.661550268610898e-05,
      "logits/chosen": -0.37981244921684265,
      "logits/rejected": -0.3839191496372223,
      "logps/chosen": -2.9031481742858887,
      "logps/rejected": -4.830349922180176,
      "loss": 0.6897,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02680964395403862,
      "rewards/margins": 0.06183745339512825,
      "rewards/rejected": -0.08864708989858627,
      "step": 870
    },
    {
      "epoch": 0.6751054852320675,
      "grad_norm": 1.763682246208191,
      "learning_rate": 1.6231772831926324e-05,
      "logits/chosen": -0.34375470876693726,
      "logits/rejected": -0.3387607932090759,
      "logps/chosen": -3.3046181201934814,
      "logps/rejected": -4.335677146911621,
      "loss": 0.6843,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04779592901468277,
      "rewards/margins": 0.02044987678527832,
      "rewards/rejected": -0.06824581325054169,
      "step": 880
    },
    {
      "epoch": 0.682777138473341,
      "grad_norm": 2.387399196624756,
      "learning_rate": 1.584804297774367e-05,
      "logits/chosen": -0.4725160598754883,
      "logits/rejected": -0.4763157367706299,
      "logps/chosen": -2.8794379234313965,
      "logps/rejected": -4.024861812591553,
      "loss": 0.6943,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.046050556004047394,
      "rewards/margins": 0.01775188557803631,
      "rewards/rejected": -0.06380244344472885,
      "step": 890
    },
    {
      "epoch": 0.6904487917146145,
      "grad_norm": 1.5189565420150757,
      "learning_rate": 1.5464313123561015e-05,
      "logits/chosen": -0.31694844365119934,
      "logits/rejected": -0.3208076059818268,
      "logps/chosen": -2.9370594024658203,
      "logps/rejected": -4.222288608551025,
      "loss": 0.6835,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.04458096995949745,
      "rewards/margins": 0.028453299775719643,
      "rewards/rejected": -0.07303427159786224,
      "step": 900
    },
    {
      "epoch": 0.6904487917146145,
      "eval_logits/chosen": -0.2631658613681793,
      "eval_logits/rejected": -0.2605777382850647,
      "eval_logps/chosen": -3.5630836486816406,
      "eval_logps/rejected": -4.000800132751465,
      "eval_loss": 0.6874542832374573,
      "eval_rewards/accuracies": 0.5093749761581421,
      "eval_rewards/chosen": -0.053020983934402466,
      "eval_rewards/margins": 0.013478020206093788,
      "eval_rewards/rejected": -0.0664990097284317,
      "eval_runtime": 87.9229,
      "eval_samples_per_second": 29.048,
      "eval_steps_per_second": 0.91,
      "step": 900
    },
    {
      "epoch": 0.698120444955888,
      "grad_norm": 1.9298661947250366,
      "learning_rate": 1.5080583269378359e-05,
      "logits/chosen": -0.4393790364265442,
      "logits/rejected": -0.43957772850990295,
      "logps/chosen": -3.8378818035125732,
      "logps/rejected": -3.808051586151123,
      "loss": 0.6941,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.08017073571681976,
      "rewards/margins": -0.014708010479807854,
      "rewards/rejected": -0.06546272337436676,
      "step": 910
    },
    {
      "epoch": 0.7057920981971615,
      "grad_norm": 1.7689329385757446,
      "learning_rate": 1.4696853415195703e-05,
      "logits/chosen": -0.46744298934936523,
      "logits/rejected": -0.4742264747619629,
      "logps/chosen": -3.0718374252319336,
      "logps/rejected": -3.5021071434020996,
      "loss": 0.6854,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.03193381056189537,
      "rewards/margins": 0.04194425791501999,
      "rewards/rejected": -0.07387807220220566,
      "step": 920
    },
    {
      "epoch": 0.713463751438435,
      "grad_norm": 1.4333994388580322,
      "learning_rate": 1.4313123561013048e-05,
      "logits/chosen": -0.3608981966972351,
      "logits/rejected": -0.3596799373626709,
      "logps/chosen": -4.751004219055176,
      "logps/rejected": -3.0089735984802246,
      "loss": 0.6926,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.0744011402130127,
      "rewards/margins": -0.03302520513534546,
      "rewards/rejected": -0.04137593135237694,
      "step": 930
    },
    {
      "epoch": 0.7211354046797085,
      "grad_norm": 1.605447769165039,
      "learning_rate": 1.3929393706830394e-05,
      "logits/chosen": -0.3481396734714508,
      "logits/rejected": -0.3457208275794983,
      "logps/chosen": -4.311774253845215,
      "logps/rejected": -3.5297610759735107,
      "loss": 0.6945,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.06438607722520828,
      "rewards/margins": -0.021948862820863724,
      "rewards/rejected": -0.04243721812963486,
      "step": 940
    },
    {
      "epoch": 0.728807057920982,
      "grad_norm": 0.9545491337776184,
      "learning_rate": 1.3545663852647736e-05,
      "logits/chosen": -0.45011836290359497,
      "logits/rejected": -0.4549293518066406,
      "logps/chosen": -2.901266098022461,
      "logps/rejected": -3.653916835784912,
      "loss": 0.6907,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.004453243222087622,
      "rewards/margins": 0.06575339287519455,
      "rewards/rejected": -0.07020662724971771,
      "step": 950
    },
    {
      "epoch": 0.728807057920982,
      "eval_logits/chosen": -0.2597009241580963,
      "eval_logits/rejected": -0.25726523995399475,
      "eval_logps/chosen": -3.299370288848877,
      "eval_logps/rejected": -3.774941921234131,
      "eval_loss": 0.6873298287391663,
      "eval_rewards/accuracies": 0.53125,
      "eval_rewards/chosen": -0.0266495980322361,
      "eval_rewards/margins": 0.01726352609694004,
      "eval_rewards/rejected": -0.04391312599182129,
      "eval_runtime": 87.9968,
      "eval_samples_per_second": 29.024,
      "eval_steps_per_second": 0.909,
      "step": 950
    },
    {
      "epoch": 0.7364787111622555,
      "grad_norm": 1.472712755203247,
      "learning_rate": 1.3161933998465081e-05,
      "logits/chosen": -0.43601498007774353,
      "logits/rejected": -0.4370816648006439,
      "logps/chosen": -2.949425220489502,
      "logps/rejected": -3.2113566398620605,
      "loss": 0.6916,
      "rewards/accuracies": 0.4000000059604645,
      "rewards/chosen": -0.03191310912370682,
      "rewards/margins": -0.030944937840104103,
      "rewards/rejected": -0.0009681672090664506,
      "step": 960
    },
    {
      "epoch": 0.744150364403529,
      "grad_norm": 1.710877537727356,
      "learning_rate": 1.2778204144282427e-05,
      "logits/chosen": -0.46664342284202576,
      "logits/rejected": -0.466810941696167,
      "logps/chosen": -3.9124972820281982,
      "logps/rejected": -3.6439247131347656,
      "loss": 0.6928,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.05568718910217285,
      "rewards/margins": -0.004351111128926277,
      "rewards/rejected": -0.05133608728647232,
      "step": 970
    },
    {
      "epoch": 0.7518220176448025,
      "grad_norm": 1.8609956502914429,
      "learning_rate": 1.239447429009977e-05,
      "logits/chosen": -0.5233210325241089,
      "logits/rejected": -0.5286127328872681,
      "logps/chosen": -3.5049729347229004,
      "logps/rejected": -2.9099135398864746,
      "loss": 0.6877,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.02429554983973503,
      "rewards/margins": 0.04005228728055954,
      "rewards/rejected": -0.06434784084558487,
      "step": 980
    },
    {
      "epoch": 0.759493670886076,
      "grad_norm": 2.3433878421783447,
      "learning_rate": 1.2010744435917114e-05,
      "logits/chosen": -0.5355673432350159,
      "logits/rejected": -0.5309178233146667,
      "logps/chosen": -2.8020567893981934,
      "logps/rejected": -4.379094123840332,
      "loss": 0.6869,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": -0.034182898700237274,
      "rewards/margins": 0.029533013701438904,
      "rewards/rejected": -0.06371591240167618,
      "step": 990
    },
    {
      "epoch": 0.7671653241273494,
      "grad_norm": 1.5511538982391357,
      "learning_rate": 1.162701458173446e-05,
      "logits/chosen": -0.41299349069595337,
      "logits/rejected": -0.4154202342033386,
      "logps/chosen": -2.535586357116699,
      "logps/rejected": -4.728541851043701,
      "loss": 0.6938,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.01752440631389618,
      "rewards/margins": 0.10527894645929337,
      "rewards/rejected": -0.12280334532260895,
      "step": 1000
    },
    {
      "epoch": 0.7671653241273494,
      "eval_logits/chosen": -0.25718361139297485,
      "eval_logits/rejected": -0.2549431622028351,
      "eval_logps/chosen": -3.5508739948272705,
      "eval_logps/rejected": -3.9987282752990723,
      "eval_loss": 0.687543511390686,
      "eval_rewards/accuracies": 0.512499988079071,
      "eval_rewards/chosen": -0.051799990236759186,
      "eval_rewards/margins": 0.014491766691207886,
      "eval_rewards/rejected": -0.06629175692796707,
      "eval_runtime": 87.5816,
      "eval_samples_per_second": 29.161,
      "eval_steps_per_second": 0.913,
      "step": 1000
    },
    {
      "epoch": 0.7748369773686229,
      "grad_norm": 1.9154669046401978,
      "learning_rate": 1.1243284727551805e-05,
      "logits/chosen": -0.5070533156394958,
      "logits/rejected": -0.5113304853439331,
      "logps/chosen": -3.3207507133483887,
      "logps/rejected": -4.22388219833374,
      "loss": 0.6931,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.0673786848783493,
      "rewards/margins": -0.0010321959853172302,
      "rewards/rejected": -0.06634648889303207,
      "step": 1010
    },
    {
      "epoch": 0.7825086306098964,
      "grad_norm": 1.3908665180206299,
      "learning_rate": 1.0859554873369149e-05,
      "logits/chosen": -0.45152488350868225,
      "logits/rejected": -0.4454428553581238,
      "logps/chosen": -4.341331481933594,
      "logps/rejected": -2.536820650100708,
      "loss": 0.688,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.0593075156211853,
      "rewards/margins": -0.003714442951604724,
      "rewards/rejected": -0.05559306591749191,
      "step": 1020
    },
    {
      "epoch": 0.7901802838511699,
      "grad_norm": 1.4446948766708374,
      "learning_rate": 1.0475825019186493e-05,
      "logits/chosen": -0.4947948455810547,
      "logits/rejected": -0.4966978132724762,
      "logps/chosen": -3.387843608856201,
      "logps/rejected": -3.6044819355010986,
      "loss": 0.6969,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.06083371490240097,
      "rewards/margins": -0.006257879547774792,
      "rewards/rejected": -0.0545758493244648,
      "step": 1030
    },
    {
      "epoch": 0.7978519370924434,
      "grad_norm": 1.610182523727417,
      "learning_rate": 1.0092095165003838e-05,
      "logits/chosen": -0.46212753653526306,
      "logits/rejected": -0.46821293234825134,
      "logps/chosen": -4.950307846069336,
      "logps/rejected": -3.0810484886169434,
      "loss": 0.69,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.04846534878015518,
      "rewards/margins": -0.025759827345609665,
      "rewards/rejected": -0.022705521434545517,
      "step": 1040
    },
    {
      "epoch": 0.8055235903337169,
      "grad_norm": 2.2419750690460205,
      "learning_rate": 9.708365310821182e-06,
      "logits/chosen": -0.5252290964126587,
      "logits/rejected": -0.5231043100357056,
      "logps/chosen": -3.5570881366729736,
      "logps/rejected": -3.7880332469940186,
      "loss": 0.6942,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06180233880877495,
      "rewards/margins": -0.026278842240571976,
      "rewards/rejected": -0.03552350401878357,
      "step": 1050
    },
    {
      "epoch": 0.8055235903337169,
      "eval_logits/chosen": -0.25850772857666016,
      "eval_logits/rejected": -0.25608521699905396,
      "eval_logps/chosen": -3.3484771251678467,
      "eval_logps/rejected": -3.820146083831787,
      "eval_loss": 0.6879977583885193,
      "eval_rewards/accuracies": 0.515625,
      "eval_rewards/chosen": -0.031560301780700684,
      "eval_rewards/margins": 0.01687331683933735,
      "eval_rewards/rejected": -0.04843362048268318,
      "eval_runtime": 87.9503,
      "eval_samples_per_second": 29.039,
      "eval_steps_per_second": 0.91,
      "step": 1050
    },
    {
      "epoch": 0.8131952435749904,
      "grad_norm": 1.5563366413116455,
      "learning_rate": 9.324635456638526e-06,
      "logits/chosen": -0.4408280849456787,
      "logits/rejected": -0.44397348165512085,
      "logps/chosen": -3.351261854171753,
      "logps/rejected": -3.4453811645507812,
      "loss": 0.6788,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.03186880797147751,
      "rewards/margins": 0.015346929430961609,
      "rewards/rejected": -0.04721573740243912,
      "step": 1060
    },
    {
      "epoch": 0.8208668968162639,
      "grad_norm": 1.7316789627075195,
      "learning_rate": 8.940905602455871e-06,
      "logits/chosen": -0.4371892511844635,
      "logits/rejected": -0.43460506200790405,
      "logps/chosen": -3.015501022338867,
      "logps/rejected": -4.129702091217041,
      "loss": 0.6886,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.010989089496433735,
      "rewards/margins": 0.06226798892021179,
      "rewards/rejected": -0.05127890035510063,
      "step": 1070
    },
    {
      "epoch": 0.8285385500575374,
      "grad_norm": 1.842515230178833,
      "learning_rate": 8.557175748273217e-06,
      "logits/chosen": -0.43732374906539917,
      "logits/rejected": -0.4389157295227051,
      "logps/chosen": -3.6136844158172607,
      "logps/rejected": -4.016729354858398,
      "loss": 0.6864,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.029665043577551842,
      "rewards/margins": 0.018349073827266693,
      "rewards/rejected": -0.04801411181688309,
      "step": 1080
    },
    {
      "epoch": 0.8362102032988109,
      "grad_norm": 1.4809995889663696,
      "learning_rate": 8.17344589409056e-06,
      "logits/chosen": -0.39032843708992004,
      "logits/rejected": -0.3862852156162262,
      "logps/chosen": -3.43218731880188,
      "logps/rejected": -3.326885223388672,
      "loss": 0.7062,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03662601485848427,
      "rewards/margins": 2.0000337826786563e-05,
      "rewards/rejected": -0.03664601594209671,
      "step": 1090
    },
    {
      "epoch": 0.8438818565400844,
      "grad_norm": 1.7941354513168335,
      "learning_rate": 7.789716039907906e-06,
      "logits/chosen": -0.3972841799259186,
      "logits/rejected": -0.4001404643058777,
      "logps/chosen": -2.768026828765869,
      "logps/rejected": -3.8738293647766113,
      "loss": 0.6784,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.028935182839632034,
      "rewards/margins": -0.008622194640338421,
      "rewards/rejected": -0.020312989130616188,
      "step": 1100
    },
    {
      "epoch": 0.8438818565400844,
      "eval_logits/chosen": -0.25824981927871704,
      "eval_logits/rejected": -0.2558724284172058,
      "eval_logps/chosen": -3.438009262084961,
      "eval_logps/rejected": -3.9306366443634033,
      "eval_loss": 0.6873763203620911,
      "eval_rewards/accuracies": 0.5218750238418579,
      "eval_rewards/chosen": -0.04051350802183151,
      "eval_rewards/margins": 0.018969083204865456,
      "eval_rewards/rejected": -0.05948259308934212,
      "eval_runtime": 103.4451,
      "eval_samples_per_second": 24.689,
      "eval_steps_per_second": 0.773,
      "step": 1100
    },
    {
      "epoch": 0.8515535097813579,
      "grad_norm": 1.5570868253707886,
      "learning_rate": 7.40598618572525e-06,
      "logits/chosen": -0.4041014611721039,
      "logits/rejected": -0.4015308916568756,
      "logps/chosen": -3.7765114307403564,
      "logps/rejected": -3.0345726013183594,
      "loss": 0.6972,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.04268242418766022,
      "rewards/margins": -0.022908544167876244,
      "rewards/rejected": -0.019773881882429123,
      "step": 1110
    },
    {
      "epoch": 0.8592251630226314,
      "grad_norm": 1.0964603424072266,
      "learning_rate": 7.022256331542594e-06,
      "logits/chosen": -0.4172094464302063,
      "logits/rejected": -0.41948825120925903,
      "logps/chosen": -3.422609329223633,
      "logps/rejected": -3.803391933441162,
      "loss": 0.6869,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.06300051510334015,
      "rewards/margins": -0.010609099641442299,
      "rewards/rejected": -0.0523914210498333,
      "step": 1120
    },
    {
      "epoch": 0.8668968162639049,
      "grad_norm": 1.906252145767212,
      "learning_rate": 6.638526477359939e-06,
      "logits/chosen": -0.44013237953186035,
      "logits/rejected": -0.4369295537471771,
      "logps/chosen": -3.6771011352539062,
      "logps/rejected": -3.834157943725586,
      "loss": 0.6724,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.033518821001052856,
      "rewards/margins": 0.04495871439576149,
      "rewards/rejected": -0.07847753912210464,
      "step": 1130
    },
    {
      "epoch": 0.8745684695051784,
      "grad_norm": 1.4220091104507446,
      "learning_rate": 6.254796623177284e-06,
      "logits/chosen": -0.5218509435653687,
      "logits/rejected": -0.527131199836731,
      "logps/chosen": -2.6902661323547363,
      "logps/rejected": -3.8175301551818848,
      "loss": 0.6918,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.05095089226961136,
      "rewards/margins": 0.008214332163333893,
      "rewards/rejected": -0.059165216982364655,
      "step": 1140
    },
    {
      "epoch": 0.8822401227464519,
      "grad_norm": 1.280017375946045,
      "learning_rate": 5.871066768994628e-06,
      "logits/chosen": -0.3222968280315399,
      "logits/rejected": -0.32637038826942444,
      "logps/chosen": -3.7984070777893066,
      "logps/rejected": -3.347282886505127,
      "loss": 0.6897,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.050425935536623,
      "rewards/margins": 0.01578127220273018,
      "rewards/rejected": -0.06620721518993378,
      "step": 1150
    },
    {
      "epoch": 0.8822401227464519,
      "eval_logits/chosen": -0.25547298789024353,
      "eval_logits/rejected": -0.25304776430130005,
      "eval_logps/chosen": -3.5789942741394043,
      "eval_logps/rejected": -4.070747375488281,
      "eval_loss": 0.6874914169311523,
      "eval_rewards/accuracies": 0.5218750238418579,
      "eval_rewards/chosen": -0.05461201071739197,
      "eval_rewards/margins": 0.01888173446059227,
      "eval_rewards/rejected": -0.07349374145269394,
      "eval_runtime": 94.5303,
      "eval_samples_per_second": 27.018,
      "eval_steps_per_second": 0.846,
      "step": 1150
    },
    {
      "epoch": 0.8899117759877253,
      "grad_norm": 1.3330215215682983,
      "learning_rate": 5.487336914811972e-06,
      "logits/chosen": -0.4289553165435791,
      "logits/rejected": -0.42481836676597595,
      "logps/chosen": -4.021355628967285,
      "logps/rejected": -2.860283374786377,
      "loss": 0.6836,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.00559669965878129,
      "rewards/margins": 0.03529663756489754,
      "rewards/rejected": -0.04089333862066269,
      "step": 1160
    },
    {
      "epoch": 0.8975834292289988,
      "grad_norm": 2.0986618995666504,
      "learning_rate": 5.103607060629317e-06,
      "logits/chosen": -0.4019424021244049,
      "logits/rejected": -0.39785686135292053,
      "logps/chosen": -2.8404293060302734,
      "logps/rejected": -5.334328651428223,
      "loss": 0.6977,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.03890450671315193,
      "rewards/margins": 0.015505614690482616,
      "rewards/rejected": -0.05441011115908623,
      "step": 1170
    },
    {
      "epoch": 0.9052550824702723,
      "grad_norm": 1.6253297328948975,
      "learning_rate": 4.719877206446662e-06,
      "logits/chosen": -0.2909601032733917,
      "logits/rejected": -0.2908070385456085,
      "logps/chosen": -2.6695282459259033,
      "logps/rejected": -4.1958699226379395,
      "loss": 0.6873,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.02150137722492218,
      "rewards/margins": 0.053162820637226105,
      "rewards/rejected": -0.07466419786214828,
      "step": 1180
    },
    {
      "epoch": 0.9129267357115458,
      "grad_norm": 1.6267880201339722,
      "learning_rate": 4.336147352264006e-06,
      "logits/chosen": -0.49156007170677185,
      "logits/rejected": -0.49580636620521545,
      "logps/chosen": -2.781247854232788,
      "logps/rejected": -4.9970293045043945,
      "loss": 0.6777,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.03273757919669151,
      "rewards/margins": 0.08275122940540314,
      "rewards/rejected": -0.11548881232738495,
      "step": 1190
    },
    {
      "epoch": 0.9205983889528193,
      "grad_norm": 1.53132164478302,
      "learning_rate": 3.952417498081351e-06,
      "logits/chosen": -0.4613899290561676,
      "logits/rejected": -0.45741963386535645,
      "logps/chosen": -3.1849541664123535,
      "logps/rejected": -4.181215763092041,
      "loss": 0.6808,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.048704423010349274,
      "rewards/margins": 0.04184052720665932,
      "rewards/rejected": -0.09054495394229889,
      "step": 1200
    },
    {
      "epoch": 0.9205983889528193,
      "eval_logits/chosen": -0.25466716289520264,
      "eval_logits/rejected": -0.2519235908985138,
      "eval_logps/chosen": -3.6220946311950684,
      "eval_logps/rejected": -4.108763694763184,
      "eval_loss": 0.6875774264335632,
      "eval_rewards/accuracies": 0.5062500238418579,
      "eval_rewards/chosen": -0.05892205238342285,
      "eval_rewards/margins": 0.018373344093561172,
      "eval_rewards/rejected": -0.07729540020227432,
      "eval_runtime": 87.859,
      "eval_samples_per_second": 29.069,
      "eval_steps_per_second": 0.911,
      "step": 1200
    },
    {
      "epoch": 0.9282700421940928,
      "grad_norm": 1.241278052330017,
      "learning_rate": 3.568687643898695e-06,
      "logits/chosen": -0.4009276032447815,
      "logits/rejected": -0.40318647027015686,
      "logps/chosen": -4.127172470092773,
      "logps/rejected": -3.866589069366455,
      "loss": 0.6829,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.09695043414831161,
      "rewards/margins": -0.046161942183971405,
      "rewards/rejected": -0.05078849941492081,
      "step": 1210
    },
    {
      "epoch": 0.9359416954353663,
      "grad_norm": 1.5091854333877563,
      "learning_rate": 3.18495778971604e-06,
      "logits/chosen": -0.4826277196407318,
      "logits/rejected": -0.478055864572525,
      "logps/chosen": -3.1896512508392334,
      "logps/rejected": -4.594376564025879,
      "loss": 0.6881,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.04869696497917175,
      "rewards/margins": 0.07252007722854614,
      "rewards/rejected": -0.12121704965829849,
      "step": 1220
    },
    {
      "epoch": 0.9436133486766398,
      "grad_norm": 1.8513944149017334,
      "learning_rate": 2.8012279355333845e-06,
      "logits/chosen": -0.44737452268600464,
      "logits/rejected": -0.4507928788661957,
      "logps/chosen": -2.7748920917510986,
      "logps/rejected": -4.892573356628418,
      "loss": 0.6837,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.0341179221868515,
      "rewards/margins": 0.06964664161205292,
      "rewards/rejected": -0.10376457124948502,
      "step": 1230
    },
    {
      "epoch": 0.9512850019179133,
      "grad_norm": 1.2888293266296387,
      "learning_rate": 2.417498081350729e-06,
      "logits/chosen": -0.42681989073753357,
      "logits/rejected": -0.4308759570121765,
      "logps/chosen": -3.6465072631835938,
      "logps/rejected": -4.087521553039551,
      "loss": 0.6922,
      "rewards/accuracies": 0.4749999940395355,
      "rewards/chosen": -0.08686716109514236,
      "rewards/margins": -0.01433576364070177,
      "rewards/rejected": -0.07253140956163406,
      "step": 1240
    },
    {
      "epoch": 0.9589566551591868,
      "grad_norm": 1.8460888862609863,
      "learning_rate": 2.0337682271680737e-06,
      "logits/chosen": -0.4428413510322571,
      "logits/rejected": -0.4398593008518219,
      "logps/chosen": -3.1560444831848145,
      "logps/rejected": -4.46377420425415,
      "loss": 0.6783,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.02867904305458069,
      "rewards/margins": 0.08603222668170929,
      "rewards/rejected": -0.11471126228570938,
      "step": 1250
    },
    {
      "epoch": 0.9589566551591868,
      "eval_logits/chosen": -0.257295697927475,
      "eval_logits/rejected": -0.25472259521484375,
      "eval_logps/chosen": -3.5708394050598145,
      "eval_logps/rejected": -4.090200424194336,
      "eval_loss": 0.6876680850982666,
      "eval_rewards/accuracies": 0.5249999761581421,
      "eval_rewards/chosen": -0.053796518594026566,
      "eval_rewards/margins": 0.021642494946718216,
      "eval_rewards/rejected": -0.07543901354074478,
      "eval_runtime": 88.4028,
      "eval_samples_per_second": 28.89,
      "eval_steps_per_second": 0.905,
      "step": 1250
    },
    {
      "epoch": 0.9666283084004603,
      "grad_norm": 2.0357563495635986,
      "learning_rate": 1.6500383729854182e-06,
      "logits/chosen": -0.40101394057273865,
      "logits/rejected": -0.40456658601760864,
      "logps/chosen": -3.5583319664001465,
      "logps/rejected": -4.374687194824219,
      "loss": 0.6872,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.056288547813892365,
      "rewards/margins": 0.033787138760089874,
      "rewards/rejected": -0.09007568657398224,
      "step": 1260
    },
    {
      "epoch": 0.9742999616417338,
      "grad_norm": 1.2816194295883179,
      "learning_rate": 1.266308518802763e-06,
      "logits/chosen": -0.45148950815200806,
      "logits/rejected": -0.4437050223350525,
      "logps/chosen": -4.794127464294434,
      "logps/rejected": -2.5109446048736572,
      "loss": 0.6883,
      "rewards/accuracies": 0.42500001192092896,
      "rewards/chosen": -0.08158718794584274,
      "rewards/margins": -0.058057576417922974,
      "rewards/rejected": -0.023529618978500366,
      "step": 1270
    },
    {
      "epoch": 0.9819716148830073,
      "grad_norm": 1.3781119585037231,
      "learning_rate": 8.825786646201075e-07,
      "logits/chosen": -0.37359991669654846,
      "logits/rejected": -0.3782901167869568,
      "logps/chosen": -3.121410369873047,
      "logps/rejected": -4.8245930671691895,
      "loss": 0.6868,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.03528415039181709,
      "rewards/margins": 0.05396044999361038,
      "rewards/rejected": -0.08924460411071777,
      "step": 1280
    },
    {
      "epoch": 0.9896432681242808,
      "grad_norm": 1.4439499378204346,
      "learning_rate": 4.988488104374521e-07,
      "logits/chosen": -0.4285728335380554,
      "logits/rejected": -0.43194809556007385,
      "logps/chosen": -4.162619113922119,
      "logps/rejected": -3.23869252204895,
      "loss": 0.687,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0661448985338211,
      "rewards/margins": -0.029600393027067184,
      "rewards/rejected": -0.036544498056173325,
      "step": 1290
    },
    {
      "epoch": 0.9973149213655543,
      "grad_norm": 1.6206567287445068,
      "learning_rate": 1.1511895625479663e-07,
      "logits/chosen": -0.4840824604034424,
      "logits/rejected": -0.4765002727508545,
      "logps/chosen": -3.34405255317688,
      "logps/rejected": -3.7568817138671875,
      "loss": 0.6942,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.05117567628622055,
      "rewards/margins": 0.012036425992846489,
      "rewards/rejected": -0.06321209669113159,
      "step": 1300
    },
    {
      "epoch": 0.9973149213655543,
      "eval_logits/chosen": -0.2575662434101105,
      "eval_logits/rejected": -0.2548866271972656,
      "eval_logps/chosen": -3.5700440406799316,
      "eval_logps/rejected": -4.087610721588135,
      "eval_loss": 0.6874552369117737,
      "eval_rewards/accuracies": 0.528124988079071,
      "eval_rewards/chosen": -0.05371702462434769,
      "eval_rewards/margins": 0.021463003009557724,
      "eval_rewards/rejected": -0.07518003135919571,
      "eval_runtime": 87.8745,
      "eval_samples_per_second": 29.064,
      "eval_steps_per_second": 0.91,
      "step": 1300
    }
  ],
  "logging_steps": 10,
  "max_steps": 1303,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}