{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.993576017130621,
  "eval_steps": 500,
  "global_step": 699,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.02,
      "grad_norm": 5.09375,
      "learning_rate": 4.999368785249369e-06,
      "log_odds_chosen": 0.06298448145389557,
      "log_odds_ratio": -0.7070032954216003,
      "logits/chosen": -2.986868143081665,
      "logits/rejected": -2.972076892852783,
      "logps/chosen": -0.7561292052268982,
      "logps/rejected": -0.8044994473457336,
      "loss": 0.6324,
      "nll_loss": 0.44739198684692383,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.15122584998607635,
      "rewards/margins": 0.00967404805123806,
      "rewards/rejected": -0.16089990735054016,
      "step": 5
    },
    {
      "epoch": 0.04,
      "grad_norm": 4.8125,
      "learning_rate": 4.997475459743124e-06,
      "log_odds_chosen": 0.0681157261133194,
      "log_odds_ratio": -0.7049038410186768,
      "logits/chosen": -2.9765102863311768,
      "logits/rejected": -2.961024045944214,
      "logps/chosen": -0.7070090174674988,
      "logps/rejected": -0.7467236518859863,
      "loss": 0.6065,
      "nll_loss": 0.4899875521659851,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.14140181243419647,
      "rewards/margins": 0.007942913100123405,
      "rewards/rejected": -0.14934472739696503,
      "step": 10
    },
    {
      "epoch": 0.06,
      "grad_norm": 3.703125,
      "learning_rate": 4.994320979557256e-06,
      "log_odds_chosen": -0.007408215198665857,
      "log_odds_ratio": -0.7562711834907532,
      "logits/chosen": -2.9262454509735107,
      "logits/rejected": -2.9222850799560547,
      "logps/chosen": -0.7819508910179138,
      "logps/rejected": -0.7833510637283325,
      "loss": 0.5555,
      "nll_loss": 0.3846359848976135,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.15639019012451172,
      "rewards/margins": 0.0002800210495479405,
      "rewards/rejected": -0.1566702127456665,
      "step": 15
    },
    {
      "epoch": 0.09,
      "grad_norm": 4.5,
      "learning_rate": 4.989906937615302e-06,
      "log_odds_chosen": 0.09043633192777634,
      "log_odds_ratio": -0.703377902507782,
      "logits/chosen": -2.915590763092041,
      "logits/rejected": -2.920872688293457,
      "logps/chosen": -0.7168866395950317,
      "logps/rejected": -0.7641353011131287,
      "loss": 0.5684,
      "nll_loss": 0.4744255542755127,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.14337733387947083,
      "rewards/margins": 0.009449719451367855,
      "rewards/rejected": -0.15282705426216125,
      "step": 20
    },
    {
      "epoch": 0.11,
      "grad_norm": 4.53125,
      "learning_rate": 4.984235562883971e-06,
      "log_odds_chosen": 0.045434046536684036,
      "log_odds_ratio": -0.72969651222229,
      "logits/chosen": -2.8840372562408447,
      "logits/rejected": -2.892764091491699,
      "logps/chosen": -0.7538484930992126,
      "logps/rejected": -0.7586053013801575,
      "loss": 0.5889,
      "nll_loss": 0.4635290503501892,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": -0.15076971054077148,
      "rewards/margins": 0.0009513672557659447,
      "rewards/rejected": -0.1517210602760315,
      "step": 25
    },
    {
      "epoch": 0.13,
      "grad_norm": 4.09375,
      "learning_rate": 4.977309719247571e-06,
      "log_odds_chosen": 0.18096226453781128,
      "log_odds_ratio": -0.6608779430389404,
      "logits/chosen": -3.06424880027771,
      "logits/rejected": -3.0758631229400635,
      "logps/chosen": -0.6707120537757874,
      "logps/rejected": -0.7431062459945679,
      "loss": 0.5515,
      "nll_loss": 0.4012710154056549,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1341424286365509,
      "rewards/margins": 0.014478841796517372,
      "rewards/rejected": -0.14862126111984253,
      "step": 30
    },
    {
      "epoch": 0.15,
      "grad_norm": 4.0625,
      "learning_rate": 4.969132904061834e-06,
      "log_odds_chosen": -0.1473471224308014,
      "log_odds_ratio": -0.812741756439209,
      "logits/chosen": -3.1558823585510254,
      "logits/rejected": -3.139392375946045,
      "logps/chosen": -0.684949517250061,
      "logps/rejected": -0.6302833557128906,
      "loss": 0.5873,
      "nll_loss": 0.3987911641597748,
      "rewards/accuracies": 0.4124999940395355,
      "rewards/chosen": -0.13698987662792206,
      "rewards/margins": -0.010933215729892254,
      "rewards/rejected": -0.12605665624141693,
      "step": 35
    },
    {
      "epoch": 0.17,
      "grad_norm": 4.21875,
      "learning_rate": 4.959709246387847e-06,
      "log_odds_chosen": 0.059796012938022614,
      "log_odds_ratio": -0.7307142019271851,
      "logits/chosen": -3.1404786109924316,
      "logits/rejected": -3.1371572017669678,
      "logps/chosen": -0.741878867149353,
      "logps/rejected": -0.7699283361434937,
      "loss": 0.6016,
      "nll_loss": 0.43490996956825256,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.14837577939033508,
      "rewards/margins": 0.005609904881566763,
      "rewards/rejected": -0.15398569405078888,
      "step": 40
    },
    {
      "epoch": 0.19,
      "grad_norm": 4.1875,
      "learning_rate": 4.9490435049069925e-06,
      "log_odds_chosen": 0.177604541182518,
      "log_odds_ratio": -0.649376630783081,
      "logits/chosen": -3.102951765060425,
      "logits/rejected": -3.1247200965881348,
      "logps/chosen": -0.6893263459205627,
      "logps/rejected": -0.7772015333175659,
      "loss": 0.5496,
      "nll_loss": 0.36714619398117065,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13786527514457703,
      "rewards/margins": 0.017575044184923172,
      "rewards/rejected": -0.1554403007030487,
      "step": 45
    },
    {
      "epoch": 0.21,
      "grad_norm": 4.0625,
      "learning_rate": 4.9371410655179495e-06,
      "log_odds_chosen": 0.12347964197397232,
      "log_odds_ratio": -0.6704575419425964,
      "logits/chosen": -3.136650562286377,
      "logits/rejected": -3.140460252761841,
      "logps/chosen": -0.6930066347122192,
      "logps/rejected": -0.7664076089859009,
      "loss": 0.5561,
      "nll_loss": 0.3619331419467926,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.13860133290290833,
      "rewards/margins": 0.014680176973342896,
      "rewards/rejected": -0.15328152477741241,
      "step": 50
    },
    {
      "epoch": 0.24,
      "grad_norm": 3.96875,
      "learning_rate": 4.924007938616967e-06,
      "log_odds_chosen": 0.08203879743814468,
      "log_odds_ratio": -0.6842275261878967,
      "logits/chosen": -3.1045684814453125,
      "logits/rejected": -3.0924317836761475,
      "logps/chosen": -0.6679835915565491,
      "logps/rejected": -0.7231112718582153,
      "loss": 0.5663,
      "nll_loss": 0.3903297185897827,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.133596733212471,
      "rewards/margins": 0.011025508865714073,
      "rewards/rejected": -0.14462223649024963,
      "step": 55
    },
    {
      "epoch": 0.26,
      "grad_norm": 5.0,
      "learning_rate": 4.909650756062782e-06,
      "log_odds_chosen": 0.21534709632396698,
      "log_odds_ratio": -0.6632246971130371,
      "logits/chosen": -3.071376323699951,
      "logits/rejected": -3.0663795471191406,
      "logps/chosen": -0.7033484578132629,
      "logps/rejected": -0.8316154479980469,
      "loss": 0.5824,
      "nll_loss": 0.4880998134613037,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.14066967368125916,
      "rewards/margins": 0.025653395801782608,
      "rewards/rejected": -0.16632308065891266,
      "step": 60
    },
    {
      "epoch": 0.28,
      "grad_norm": 4.875,
      "learning_rate": 4.894076767827721e-06,
      "log_odds_chosen": 0.07506684958934784,
      "log_odds_ratio": -0.7063411474227905,
      "logits/chosen": -3.1346957683563232,
      "logits/rejected": -3.1342291831970215,
      "logps/chosen": -0.6925743222236633,
      "logps/rejected": -0.740242600440979,
      "loss": 0.6192,
      "nll_loss": 0.46551480889320374,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13851484656333923,
      "rewards/margins": 0.009533658623695374,
      "rewards/rejected": -0.1480485200881958,
      "step": 65
    },
    {
      "epoch": 0.3,
      "grad_norm": 4.59375,
      "learning_rate": 4.8772938383366615e-06,
      "log_odds_chosen": 0.020953459665179253,
      "log_odds_ratio": -0.7297095060348511,
      "logits/chosen": -3.1903274059295654,
      "logits/rejected": -3.2007622718811035,
      "logps/chosen": -0.6915528774261475,
      "logps/rejected": -0.717432975769043,
      "loss": 0.5988,
      "nll_loss": 0.43604594469070435,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13831056654453278,
      "rewards/margins": 0.005176021717488766,
      "rewards/rejected": -0.14348658919334412,
      "step": 70
    },
    {
      "epoch": 0.32,
      "grad_norm": 3.859375,
      "learning_rate": 4.8593104424957275e-06,
      "log_odds_chosen": 0.060929737985134125,
      "log_odds_ratio": -0.714811384677887,
      "logits/chosen": -3.196502923965454,
      "logits/rejected": -3.1885693073272705,
      "logps/chosen": -0.6992107629776001,
      "logps/rejected": -0.7475954294204712,
      "loss": 0.5942,
      "nll_loss": 0.4152112603187561,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1398421823978424,
      "rewards/margins": 0.009676935151219368,
      "rewards/rejected": -0.14951911568641663,
      "step": 75
    },
    {
      "epoch": 0.34,
      "grad_norm": 4.53125,
      "learning_rate": 4.840135661412696e-06,
      "log_odds_chosen": 0.24067020416259766,
      "log_odds_ratio": -0.6120481491088867,
      "logits/chosen": -3.169623851776123,
      "logits/rejected": -3.183824062347412,
      "logps/chosen": -0.6291205883026123,
      "logps/rejected": -0.7557007670402527,
      "loss": 0.5789,
      "nll_loss": 0.3959899842739105,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.12582412362098694,
      "rewards/margins": 0.025316035374999046,
      "rewards/rejected": -0.15114016830921173,
      "step": 80
    },
    {
      "epoch": 0.36,
      "grad_norm": 4.40625,
      "learning_rate": 4.819779177811294e-06,
      "log_odds_chosen": 0.1330593079328537,
      "log_odds_ratio": -0.6760435700416565,
      "logits/chosen": -3.2041194438934326,
      "logits/rejected": -3.1831653118133545,
      "logps/chosen": -0.6773617267608643,
      "logps/rejected": -0.7431901693344116,
      "loss": 0.5506,
      "nll_loss": 0.39580783247947693,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1354723572731018,
      "rewards/margins": 0.013165691867470741,
      "rewards/rejected": -0.1486380398273468,
      "step": 85
    },
    {
      "epoch": 0.39,
      "grad_norm": 3.890625,
      "learning_rate": 4.7982512711416995e-06,
      "log_odds_chosen": 0.16101466119289398,
      "log_odds_ratio": -0.676543116569519,
      "logits/chosen": -3.1818435192108154,
      "logits/rejected": -3.1896145343780518,
      "logps/chosen": -0.7060041427612305,
      "logps/rejected": -0.8224822878837585,
      "loss": 0.5841,
      "nll_loss": 0.4883156418800354,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.14120084047317505,
      "rewards/margins": 0.023295629769563675,
      "rewards/rejected": -0.16449645161628723,
      "step": 90
    },
    {
      "epoch": 0.41,
      "grad_norm": 4.25,
      "learning_rate": 4.775562812389704e-06,
      "log_odds_chosen": 0.1345851719379425,
      "log_odds_ratio": -0.6816418766975403,
      "logits/chosen": -3.2042739391326904,
      "logits/rejected": -3.2034194469451904,
      "logps/chosen": -0.6822711229324341,
      "logps/rejected": -0.7532314658164978,
      "loss": 0.5903,
      "nll_loss": 0.4755779802799225,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13645422458648682,
      "rewards/margins": 0.014192057773470879,
      "rewards/rejected": -0.15064629912376404,
      "step": 95
    },
    {
      "epoch": 0.43,
      "grad_norm": 3.71875,
      "learning_rate": 4.751725258587172e-06,
      "log_odds_chosen": 0.14706887304782867,
      "log_odds_ratio": -0.6651312112808228,
      "logits/chosen": -3.1903748512268066,
      "logits/rejected": -3.1974258422851562,
      "logps/chosen": -0.6400434374809265,
      "logps/rejected": -0.7148961424827576,
      "loss": 0.5851,
      "nll_loss": 0.4396067261695862,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.12800869345664978,
      "rewards/margins": 0.014970543794333935,
      "rewards/rejected": -0.142979234457016,
      "step": 100
    },
    {
      "epoch": 0.45,
      "grad_norm": 3.890625,
      "learning_rate": 4.726750647026569e-06,
      "log_odds_chosen": 0.03515060991048813,
      "log_odds_ratio": -0.7285342216491699,
      "logits/chosen": -3.110316514968872,
      "logits/rejected": -3.1469106674194336,
      "logps/chosen": -0.7531843781471252,
      "logps/rejected": -0.7670533061027527,
      "loss": 0.585,
      "nll_loss": 0.46621793508529663,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.15063688158988953,
      "rewards/margins": 0.0027738078497350216,
      "rewards/rejected": -0.1534106731414795,
      "step": 105
    },
    {
      "epoch": 0.47,
      "grad_norm": 4.0,
      "learning_rate": 4.700651589182461e-06,
      "log_odds_chosen": 0.11814375966787338,
      "log_odds_ratio": -0.6774541735649109,
      "logits/chosen": -3.1227428913116455,
      "logits/rejected": -3.158555269241333,
      "logps/chosen": -0.7057517170906067,
      "logps/rejected": -0.7718692421913147,
      "loss": 0.5403,
      "nll_loss": 0.37699857354164124,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.1411503553390503,
      "rewards/margins": 0.013223511166870594,
      "rewards/rejected": -0.15437385439872742,
      "step": 110
    },
    {
      "epoch": 0.49,
      "grad_norm": 4.0625,
      "learning_rate": 4.6734412643430795e-06,
      "log_odds_chosen": 0.15334436297416687,
      "log_odds_ratio": -0.6591517329216003,
      "logits/chosen": -3.152284622192383,
      "logits/rejected": -3.137619972229004,
      "logps/chosen": -0.6621846556663513,
      "logps/rejected": -0.7469123005867004,
      "loss": 0.5819,
      "nll_loss": 0.4604349136352539,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13243696093559265,
      "rewards/margins": 0.01694551296532154,
      "rewards/rejected": -0.14938244223594666,
      "step": 115
    },
    {
      "epoch": 0.51,
      "grad_norm": 3.6875,
      "learning_rate": 4.64513341295515e-06,
      "log_odds_chosen": 0.25181251764297485,
      "log_odds_ratio": -0.6330325603485107,
      "logits/chosen": -3.1038594245910645,
      "logits/rejected": -3.127596139907837,
      "logps/chosen": -0.6145869493484497,
      "logps/rejected": -0.7382071614265442,
      "loss": 0.5614,
      "nll_loss": 0.4344588816165924,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12291739135980606,
      "rewards/margins": 0.024724040180444717,
      "rewards/rejected": -0.14764143526554108,
      "step": 120
    },
    {
      "epoch": 0.54,
      "grad_norm": 4.5625,
      "learning_rate": 4.615742329685358e-06,
      "log_odds_chosen": 0.06189741939306259,
      "log_odds_ratio": -0.7182206511497498,
      "logits/chosen": -3.124662399291992,
      "logits/rejected": -3.129429340362549,
      "logps/chosen": -0.7151820063591003,
      "logps/rejected": -0.7391675710678101,
      "loss": 0.5781,
      "nll_loss": 0.42448896169662476,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1430363953113556,
      "rewards/margins": 0.004797111731022596,
      "rewards/rejected": -0.14783351123332977,
      "step": 125
    },
    {
      "epoch": 0.56,
      "grad_norm": 4.1875,
      "learning_rate": 4.58528285620194e-06,
      "log_odds_chosen": 0.15051795542240143,
      "log_odds_ratio": -0.6667842864990234,
      "logits/chosen": -3.1157212257385254,
      "logits/rejected": -3.131880521774292,
      "logps/chosen": -0.6726551651954651,
      "logps/rejected": -0.7339776754379272,
      "loss": 0.5517,
      "nll_loss": 0.3924528658390045,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.13453105092048645,
      "rewards/margins": 0.012264521792531013,
      "rewards/rejected": -0.14679554104804993,
      "step": 130
    },
    {
      "epoch": 0.58,
      "grad_norm": 3.8125,
      "learning_rate": 4.553770373680062e-06,
      "log_odds_chosen": 0.03864496201276779,
      "log_odds_ratio": -0.738520085811615,
      "logits/chosen": -3.108069896697998,
      "logits/rejected": -3.111837148666382,
      "logps/chosen": -0.6787298321723938,
      "logps/rejected": -0.7142878770828247,
      "loss": 0.5869,
      "nll_loss": 0.36721861362457275,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.13574595749378204,
      "rewards/margins": 0.007111614104360342,
      "rewards/rejected": -0.14285758137702942,
      "step": 135
    },
    {
      "epoch": 0.6,
      "grad_norm": 4.21875,
      "learning_rate": 4.521220795034763e-06,
      "log_odds_chosen": 0.15647068619728088,
      "log_odds_ratio": -0.673542857170105,
      "logits/chosen": -3.060241222381592,
      "logits/rejected": -3.0633511543273926,
      "logps/chosen": -0.6453939080238342,
      "logps/rejected": -0.7274571657180786,
      "loss": 0.5505,
      "nll_loss": 0.41087573766708374,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12907877564430237,
      "rewards/margins": 0.01641263999044895,
      "rewards/rejected": -0.14549142122268677,
      "step": 140
    },
    {
      "epoch": 0.62,
      "grad_norm": 4.28125,
      "learning_rate": 4.487650556885378e-06,
      "log_odds_chosen": 0.13851861655712128,
      "log_odds_ratio": -0.6815603971481323,
      "logits/chosen": -3.110565185546875,
      "logits/rejected": -3.108269691467285,
      "logps/chosen": -0.7905935049057007,
      "logps/rejected": -0.863776683807373,
      "loss": 0.5402,
      "nll_loss": 0.4115132689476013,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.15811872482299805,
      "rewards/margins": 0.014636626467108727,
      "rewards/rejected": -0.17275534570217133,
      "step": 145
    },
    {
      "epoch": 0.64,
      "grad_norm": 4.03125,
      "learning_rate": 4.453076611255507e-06,
      "log_odds_chosen": 0.1597544103860855,
      "log_odds_ratio": -0.6715651750564575,
      "logits/chosen": -3.1071646213531494,
      "logits/rejected": -3.0957367420196533,
      "logps/chosen": -0.6362262964248657,
      "logps/rejected": -0.7219537496566772,
      "loss": 0.5297,
      "nll_loss": 0.38165420293807983,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.12724526226520538,
      "rewards/margins": 0.01714547723531723,
      "rewards/rejected": -0.14439073204994202,
      "step": 150
    },
    {
      "epoch": 0.66,
      "grad_norm": 4.21875,
      "learning_rate": 4.417516417012725e-06,
      "log_odds_chosen": -0.022897567600011826,
      "log_odds_ratio": -0.7432743310928345,
      "logits/chosen": -3.0967166423797607,
      "logits/rejected": -3.0849483013153076,
      "logps/chosen": -0.703900933265686,
      "logps/rejected": -0.6957621574401855,
      "loss": 0.5717,
      "nll_loss": 0.43085795640945435,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.14078018069267273,
      "rewards/margins": -0.0016277560498565435,
      "rewards/rejected": -0.1391524374485016,
      "step": 155
    },
    {
      "epoch": 0.69,
      "grad_norm": 3.90625,
      "learning_rate": 4.380987931052347e-06,
      "log_odds_chosen": 0.12808199226856232,
      "log_odds_ratio": -0.683525562286377,
      "logits/chosen": -3.105473041534424,
      "logits/rejected": -3.1155142784118652,
      "logps/chosen": -0.7174890637397766,
      "logps/rejected": -0.7977532148361206,
      "loss": 0.5653,
      "nll_loss": 0.37563520669937134,
      "rewards/accuracies": 0.550000011920929,
      "rewards/chosen": -0.14349780976772308,
      "rewards/margins": 0.016052838414907455,
      "rewards/rejected": -0.15955063700675964,
      "step": 160
    },
    {
      "epoch": 0.71,
      "grad_norm": 4.28125,
      "learning_rate": 4.343509599229697e-06,
      "log_odds_chosen": 0.07662512362003326,
      "log_odds_ratio": -0.7142797708511353,
      "logits/chosen": -3.1126039028167725,
      "logits/rejected": -3.1231586933135986,
      "logps/chosen": -0.6783756017684937,
      "logps/rejected": -0.7291387319564819,
      "loss": 0.5674,
      "nll_loss": 0.4361972212791443,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.1356751173734665,
      "rewards/margins": 0.010152625851333141,
      "rewards/rejected": -0.1458277702331543,
      "step": 165
    },
    {
      "epoch": 0.73,
      "grad_norm": 3.828125,
      "learning_rate": 4.305100347045476e-06,
      "log_odds_chosen": 0.1829986721277237,
      "log_odds_ratio": -0.6602972745895386,
      "logits/chosen": -3.135678768157959,
      "logits/rejected": -3.1565449237823486,
      "logps/chosen": -0.6898233890533447,
      "logps/rejected": -0.7847301363945007,
      "loss": 0.548,
      "nll_loss": 0.3952903151512146,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.13796468079090118,
      "rewards/margins": 0.018981358036398888,
      "rewards/rejected": -0.156946063041687,
      "step": 170
    },
    {
      "epoch": 0.75,
      "grad_norm": 4.46875,
      "learning_rate": 4.265779570088914e-06,
      "log_odds_chosen": 0.23545940220355988,
      "log_odds_ratio": -0.6459792852401733,
      "logits/chosen": -3.142263174057007,
      "logits/rejected": -3.1482534408569336,
      "logps/chosen": -0.6683803796768188,
      "logps/rejected": -0.8047019839286804,
      "loss": 0.5477,
      "nll_loss": 0.4291974604129791,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.13367608189582825,
      "rewards/margins": 0.02726433239877224,
      "rewards/rejected": -0.16094039380550385,
      "step": 175
    },
    {
      "epoch": 0.77,
      "grad_norm": 4.59375,
      "learning_rate": 4.22556712424355e-06,
      "log_odds_chosen": 0.13412100076675415,
      "log_odds_ratio": -0.678640604019165,
      "logits/chosen": -3.165506601333618,
      "logits/rejected": -3.1562082767486572,
      "logps/chosen": -0.696603000164032,
      "logps/rejected": -0.7526296377182007,
      "loss": 0.5847,
      "nll_loss": 0.383400559425354,
      "rewards/accuracies": 0.612500011920929,
      "rewards/chosen": -0.13932058215141296,
      "rewards/margins": 0.011205337010324001,
      "rewards/rejected": -0.15052591264247894,
      "step": 180
    },
    {
      "epoch": 0.79,
      "grad_norm": 3.71875,
      "learning_rate": 4.184483315660565e-06,
      "log_odds_chosen": 0.20085343718528748,
      "log_odds_ratio": -0.6461249589920044,
      "logits/chosen": -3.1610589027404785,
      "logits/rejected": -3.1671133041381836,
      "logps/chosen": -0.6673001050949097,
      "logps/rejected": -0.7822000980377197,
      "loss": 0.5358,
      "nll_loss": 0.4822109341621399,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.13346004486083984,
      "rewards/margins": 0.022979993373155594,
      "rewards/rejected": -0.15644003450870514,
      "step": 185
    },
    {
      "epoch": 0.81,
      "grad_norm": 3.625,
      "learning_rate": 4.1425488905047485e-06,
      "log_odds_chosen": 0.06839115917682648,
      "log_odds_ratio": -0.7076748609542847,
      "logits/chosen": -3.1488382816314697,
      "logits/rejected": -3.1493427753448486,
      "logps/chosen": -0.6919032335281372,
      "logps/rejected": -0.7248092293739319,
      "loss": 0.5477,
      "nll_loss": 0.4083670973777771,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13838066160678864,
      "rewards/margins": 0.006581192370504141,
      "rewards/rejected": -0.14496184885501862,
      "step": 190
    },
    {
      "epoch": 0.84,
      "grad_norm": 4.15625,
      "learning_rate": 4.099785024478276e-06,
      "log_odds_chosen": 0.2139725238084793,
      "log_odds_ratio": -0.628895103931427,
      "logits/chosen": -3.124779224395752,
      "logits/rejected": -3.1284875869750977,
      "logps/chosen": -0.5759474635124207,
      "logps/rejected": -0.6854532361030579,
      "loss": 0.5441,
      "nll_loss": 0.37735462188720703,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.11518951505422592,
      "rewards/margins": 0.021901141852140427,
      "rewards/rejected": -0.13709065318107605,
      "step": 195
    },
    {
      "epoch": 0.86,
      "grad_norm": 3.78125,
      "learning_rate": 4.056213312127573e-06,
      "log_odds_chosen": 0.18503764271736145,
      "log_odds_ratio": -0.6680355668067932,
      "logits/chosen": -3.078535556793213,
      "logits/rejected": -3.0459985733032227,
      "logps/chosen": -0.747636079788208,
      "logps/rejected": -0.8532911539077759,
      "loss": 0.5496,
      "nll_loss": 0.39417606592178345,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.14952722191810608,
      "rewards/margins": 0.021131008863449097,
      "rewards/rejected": -0.1706582009792328,
      "step": 200
    },
    {
      "epoch": 0.88,
      "grad_norm": 3.625,
      "learning_rate": 4.011855755938674e-06,
      "log_odds_chosen": 0.01333926897495985,
      "log_odds_ratio": -0.7361116409301758,
      "logits/chosen": -3.0914862155914307,
      "logits/rejected": -3.105278253555298,
      "logps/chosen": -0.6985930800437927,
      "logps/rejected": -0.6959635615348816,
      "loss": 0.5328,
      "nll_loss": 0.3898618221282959,
      "rewards/accuracies": 0.512499988079071,
      "rewards/chosen": -0.13971860706806183,
      "rewards/margins": -0.0005258865421637893,
      "rewards/rejected": -0.13919273018836975,
      "step": 205
    },
    {
      "epoch": 0.9,
      "grad_norm": 3.9375,
      "learning_rate": 3.9667347552265945e-06,
      "log_odds_chosen": 0.10643716901540756,
      "log_odds_ratio": -0.6941254734992981,
      "logits/chosen": -3.106863021850586,
      "logits/rejected": -3.1205527782440186,
      "logps/chosen": -0.6641048192977905,
      "logps/rejected": -0.705162525177002,
      "loss": 0.5271,
      "nll_loss": 0.3007195293903351,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.1328209638595581,
      "rewards/margins": 0.008211550302803516,
      "rewards/rejected": -0.14103251695632935,
      "step": 210
    },
    {
      "epoch": 0.92,
      "grad_norm": 3.703125,
      "learning_rate": 3.920873094824305e-06,
      "log_odds_chosen": 0.1166054978966713,
      "log_odds_ratio": -0.6895023584365845,
      "logits/chosen": -3.114652633666992,
      "logits/rejected": -3.115201473236084,
      "logps/chosen": -0.7006415724754333,
      "logps/rejected": -0.7791968584060669,
      "loss": 0.5836,
      "nll_loss": 0.42902565002441406,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.14012829959392548,
      "rewards/margins": 0.015711039304733276,
      "rewards/rejected": -0.15583935379981995,
      "step": 215
    },
    {
      "epoch": 0.94,
      "grad_norm": 3.84375,
      "learning_rate": 3.874293933577034e-06,
      "log_odds_chosen": 0.10644589364528656,
      "log_odds_ratio": -0.6865199208259583,
      "logits/chosen": -3.119800090789795,
      "logits/rejected": -3.135976791381836,
      "logps/chosen": -0.693631649017334,
      "logps/rejected": -0.7763968706130981,
      "loss": 0.557,
      "nll_loss": 0.3763471245765686,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.13872632384300232,
      "rewards/margins": 0.016553033143281937,
      "rewards/rejected": -0.15527936816215515,
      "step": 220
    },
    {
      "epoch": 0.96,
      "grad_norm": 3.859375,
      "learning_rate": 3.8270207926477e-06,
      "log_odds_chosen": 0.06535528600215912,
      "log_odds_ratio": -0.7187795639038086,
      "logits/chosen": -3.1649396419525146,
      "logits/rejected": -3.150834560394287,
      "logps/chosen": -0.6850636601448059,
      "logps/rejected": -0.7298869490623474,
      "loss": 0.5689,
      "nll_loss": 0.39077669382095337,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": -0.13701273500919342,
      "rewards/margins": 0.00896465964615345,
      "rewards/rejected": -0.14597737789154053,
      "step": 225
    },
    {
      "epoch": 0.99,
      "grad_norm": 4.21875,
      "learning_rate": 3.7790775436393954e-06,
      "log_odds_chosen": 0.11255147308111191,
      "log_odds_ratio": -0.6830418705940247,
      "logits/chosen": -3.0758774280548096,
      "logits/rejected": -3.0805001258850098,
      "logps/chosen": -0.6779443621635437,
      "logps/rejected": -0.7496141791343689,
      "loss": 0.5525,
      "nll_loss": 0.413738876581192,
      "rewards/accuracies": 0.6000000238418579,
      "rewards/chosen": -0.1355888843536377,
      "rewards/margins": 0.014333980157971382,
      "rewards/rejected": -0.14992286264896393,
      "step": 230
    },
    {
      "epoch": 1.01,
      "grad_norm": 4.09375,
      "learning_rate": 3.7304883965408944e-06,
      "log_odds_chosen": 0.25222933292388916,
      "log_odds_ratio": -0.6448281407356262,
      "logits/chosen": -3.1005845069885254,
      "logits/rejected": -3.1040127277374268,
      "logps/chosen": -0.6791092157363892,
      "logps/rejected": -0.7944923639297485,
      "loss": 0.5541,
      "nll_loss": 0.43306249380111694,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.1358218491077423,
      "rewards/margins": 0.023076623678207397,
      "rewards/rejected": -0.1588984727859497,
      "step": 235
    },
    {
      "epoch": 1.03,
      "grad_norm": 4.5,
      "learning_rate": 3.68127788750129e-06,
      "log_odds_chosen": 0.862420380115509,
      "log_odds_ratio": -0.4000455439090729,
      "logits/chosen": -3.131739854812622,
      "logits/rejected": -3.1033473014831543,
      "logps/chosen": -0.48904451727867126,
      "logps/rejected": -0.9010938405990601,
      "loss": 0.4516,
      "nll_loss": 0.37263303995132446,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.09780890494585037,
      "rewards/margins": 0.08240985870361328,
      "rewards/rejected": -0.18021877110004425,
      "step": 240
    },
    {
      "epoch": 1.05,
      "grad_norm": 3.875,
      "learning_rate": 3.6314708664399378e-06,
      "log_odds_chosen": 0.8805850744247437,
      "log_odds_ratio": -0.39087027311325073,
      "logits/chosen": -3.1137661933898926,
      "logits/rejected": -3.133709669113159,
      "logps/chosen": -0.468756765127182,
      "logps/rejected": -0.9099340438842773,
      "loss": 0.4385,
      "nll_loss": 0.33198338747024536,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.09375137090682983,
      "rewards/margins": 0.08823545277118683,
      "rewards/rejected": -0.18198682367801666,
      "step": 245
    },
    {
      "epoch": 1.07,
      "grad_norm": 4.09375,
      "learning_rate": 3.581092484497941e-06,
      "log_odds_chosen": 0.9555109739303589,
      "log_odds_ratio": -0.3677138686180115,
      "logits/chosen": -3.079671859741211,
      "logits/rejected": -3.095301866531372,
      "logps/chosen": -0.46135392785072327,
      "logps/rejected": -0.900970458984375,
      "loss": 0.4354,
      "nll_loss": 0.3639451861381531,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09227079153060913,
      "rewards/margins": 0.08792328834533691,
      "rewards/rejected": -0.18019406497478485,
      "step": 250
    },
    {
      "epoch": 1.09,
      "grad_norm": 4.6875,
      "learning_rate": 3.5301681813375343e-06,
      "log_odds_chosen": 0.8514328002929688,
      "log_odds_ratio": -0.40046748518943787,
      "logits/chosen": -3.065979480743408,
      "logits/rejected": -3.0509727001190186,
      "logps/chosen": -0.5572352409362793,
      "logps/rejected": -0.9676912426948547,
      "loss": 0.4566,
      "nll_loss": 0.3993757665157318,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1114470362663269,
      "rewards/margins": 0.08209122717380524,
      "rewards/rejected": -0.19353827834129333,
      "step": 255
    },
    {
      "epoch": 1.11,
      "grad_norm": 4.59375,
      "learning_rate": 3.4787236722957747e-06,
      "log_odds_chosen": 0.9488438367843628,
      "log_odds_ratio": -0.39394739270210266,
      "logits/chosen": -3.103703260421753,
      "logits/rejected": -3.120194435119629,
      "logps/chosen": -0.5076486468315125,
      "logps/rejected": -0.9956499934196472,
      "loss": 0.4487,
      "nll_loss": 0.3443233370780945,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10152973979711533,
      "rewards/margins": 0.09760028123855591,
      "rewards/rejected": -0.19913002848625183,
      "step": 260
    },
    {
      "epoch": 1.13,
      "grad_norm": 4.53125,
      "learning_rate": 3.4267849353990178e-06,
      "log_odds_chosen": 1.0445611476898193,
      "log_odds_ratio": -0.3562309145927429,
      "logits/chosen": -3.0731451511383057,
      "logits/rejected": -3.0952889919281006,
      "logps/chosen": -0.5211637020111084,
      "logps/rejected": -1.046607494354248,
      "loss": 0.4208,
      "nll_loss": 0.3219150900840759,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10423272848129272,
      "rewards/margins": 0.10508877038955688,
      "rewards/rejected": -0.2093214988708496,
      "step": 265
    },
    {
      "epoch": 1.16,
      "grad_norm": 4.46875,
      "learning_rate": 3.3743781982447533e-06,
      "log_odds_chosen": 1.1367815732955933,
      "log_odds_ratio": -0.34295934438705444,
      "logits/chosen": -3.10606050491333,
      "logits/rejected": -3.115898609161377,
      "logps/chosen": -0.4709681570529938,
      "logps/rejected": -1.036333441734314,
      "loss": 0.4494,
      "nll_loss": 0.37718456983566284,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09419362246990204,
      "rewards/margins": 0.11307307332754135,
      "rewards/rejected": -0.2072666883468628,
      "step": 270
    },
    {
      "epoch": 1.18,
      "grad_norm": 4.71875,
      "learning_rate": 3.3215299247573974e-06,
      "log_odds_chosen": 0.904280960559845,
      "log_odds_ratio": -0.3935766816139221,
      "logits/chosen": -3.0998826026916504,
      "logits/rejected": -3.12553071975708,
      "logps/chosen": -0.5041223168373108,
      "logps/rejected": -0.9397950172424316,
      "loss": 0.4368,
      "nll_loss": 0.38810276985168457,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10082447528839111,
      "rewards/margins": 0.08713451027870178,
      "rewards/rejected": -0.1879589855670929,
      "step": 275
    },
    {
      "epoch": 1.2,
      "grad_norm": 5.34375,
      "learning_rate": 3.268266801824768e-06,
      "log_odds_chosen": 1.1236122846603394,
      "log_odds_ratio": -0.3298667073249817,
      "logits/chosen": -3.0786471366882324,
      "logits/rejected": -3.088914632797241,
      "logps/chosen": -0.49954742193222046,
      "logps/rejected": -1.0479991436004639,
      "loss": 0.4035,
      "nll_loss": 0.35881948471069336,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09990949928760529,
      "rewards/margins": 0.1096903458237648,
      "rewards/rejected": -0.20959985256195068,
      "step": 280
    },
    {
      "epoch": 1.22,
      "grad_norm": 4.5625,
      "learning_rate": 3.2146157258219534e-06,
      "log_odds_chosen": 0.9877004623413086,
      "log_odds_ratio": -0.36029869318008423,
      "logits/chosen": -3.104013204574585,
      "logits/rejected": -3.1102428436279297,
      "logps/chosen": -0.4857029318809509,
      "logps/rejected": -0.9460722804069519,
      "loss": 0.4266,
      "nll_loss": 0.31352436542510986,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.0971405953168869,
      "rewards/margins": 0.09207387268543243,
      "rewards/rejected": -0.18921445310115814,
      "step": 285
    },
    {
      "epoch": 1.24,
      "grad_norm": 4.6875,
      "learning_rate": 3.1606037890294013e-06,
      "log_odds_chosen": 0.9617363214492798,
      "log_odds_ratio": -0.36377251148223877,
      "logits/chosen": -3.1021533012390137,
      "logits/rejected": -3.1139049530029297,
      "logps/chosen": -0.5411451458930969,
      "logps/rejected": -1.0121533870697021,
      "loss": 0.4047,
      "nll_loss": 0.3547790050506592,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10822904109954834,
      "rewards/margins": 0.09420166909694672,
      "rewards/rejected": -0.20243068039417267,
      "step": 290
    },
    {
      "epoch": 1.26,
      "grad_norm": 4.5625,
      "learning_rate": 3.106258265952082e-06,
      "log_odds_chosen": 0.9374507665634155,
      "log_odds_ratio": -0.3822116255760193,
      "logits/chosen": -3.1063599586486816,
      "logits/rejected": -3.102320432662964,
      "logps/chosen": -0.5152262449264526,
      "logps/rejected": -0.9716517329216003,
      "loss": 0.4226,
      "nll_loss": 0.3514345586299896,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10304524749517441,
      "rewards/margins": 0.0912851020693779,
      "rewards/rejected": -0.1943303644657135,
      "step": 295
    },
    {
      "epoch": 1.28,
      "grad_norm": 4.6875,
      "learning_rate": 3.0516065995466336e-06,
      "log_odds_chosen": 1.088078260421753,
      "log_odds_ratio": -0.3645312786102295,
      "logits/chosen": -3.108431100845337,
      "logits/rejected": -3.1180756092071533,
      "logps/chosen": -0.4901120066642761,
      "logps/rejected": -1.0360512733459473,
      "loss": 0.4289,
      "nll_loss": 0.3283267021179199,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.09802242368459702,
      "rewards/margins": 0.10918784141540527,
      "rewards/rejected": -0.2072102576494217,
      "step": 300
    },
    {
      "epoch": 1.31,
      "grad_norm": 4.40625,
      "learning_rate": 2.9966763873634424e-06,
      "log_odds_chosen": 1.2089542150497437,
      "log_odds_ratio": -0.3031514286994934,
      "logits/chosen": -3.1029577255249023,
      "logits/rejected": -3.1095030307769775,
      "logps/chosen": -0.48879045248031616,
      "logps/rejected": -1.1196680068969727,
      "loss": 0.4317,
      "nll_loss": 0.3550404906272888,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.09775808453559875,
      "rewards/margins": 0.12617555260658264,
      "rewards/rejected": -0.2239336222410202,
      "step": 305
    },
    {
      "epoch": 1.33,
      "grad_norm": 4.5,
      "learning_rate": 2.941495367610656e-06,
      "log_odds_chosen": 1.095005989074707,
      "log_odds_ratio": -0.3307039439678192,
      "logits/chosen": -3.049807071685791,
      "logits/rejected": -3.064573049545288,
      "logps/chosen": -0.45625075697898865,
      "logps/rejected": -0.9708633422851562,
      "loss": 0.4099,
      "nll_loss": 0.32795679569244385,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.09125015884637833,
      "rewards/margins": 0.10292251408100128,
      "rewards/rejected": -0.194172665476799,
      "step": 310
    },
    {
      "epoch": 1.35,
      "grad_norm": 6.03125,
      "learning_rate": 2.8860914051471722e-06,
      "log_odds_chosen": 0.9841387867927551,
      "log_odds_ratio": -0.3754274845123291,
      "logits/chosen": -3.06512713432312,
      "logits/rejected": -3.047639846801758,
      "logps/chosen": -0.5017204284667969,
      "logps/rejected": -0.9973098635673523,
      "loss": 0.4311,
      "nll_loss": 0.36222928762435913,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10034408420324326,
      "rewards/margins": 0.09911789745092392,
      "rewards/rejected": -0.19946198165416718,
      "step": 315
    },
    {
      "epoch": 1.37,
      "grad_norm": 4.375,
      "learning_rate": 2.830492477411671e-06,
      "log_odds_chosen": 1.045000433921814,
      "log_odds_ratio": -0.37661224603652954,
      "logits/chosen": -3.064300537109375,
      "logits/rejected": -3.076216697692871,
      "logps/chosen": -0.5427008867263794,
      "logps/rejected": -1.0882201194763184,
      "loss": 0.4399,
      "nll_loss": 0.3907185196876526,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.10854017734527588,
      "rewards/margins": 0.10910383611917496,
      "rewards/rejected": -0.21764402091503143,
      "step": 320
    },
    {
      "epoch": 1.39,
      "grad_norm": 4.96875,
      "learning_rate": 2.774726660294799e-06,
      "log_odds_chosen": 1.176218032836914,
      "log_odds_ratio": -0.32928329706192017,
      "logits/chosen": -3.094344139099121,
      "logits/rejected": -3.099308967590332,
      "logps/chosen": -0.5172286033630371,
      "logps/rejected": -1.1497700214385986,
      "loss": 0.4094,
      "nll_loss": 0.30435997247695923,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.10344573110342026,
      "rewards/margins": 0.12650831043720245,
      "rewards/rejected": -0.22995400428771973,
      "step": 325
    },
    {
      "epoch": 1.41,
      "grad_norm": 11.0,
      "learning_rate": 2.7188221139616303e-06,
      "log_odds_chosen": 1.014714002609253,
      "log_odds_ratio": -0.3674837648868561,
      "logits/chosen": -3.068471670150757,
      "logits/rejected": -3.0866987705230713,
      "logps/chosen": -0.48677653074264526,
      "logps/rejected": -0.9665210843086243,
      "loss": 0.4141,
      "nll_loss": 0.36226484179496765,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.09735531359910965,
      "rewards/margins": 0.09594890475273132,
      "rewards/rejected": -0.19330421090126038,
      "step": 330
    },
    {
      "epoch": 1.43,
      "grad_norm": 4.875,
      "learning_rate": 2.662807068631585e-06,
      "log_odds_chosen": 1.0920135974884033,
      "log_odds_ratio": -0.36240848898887634,
      "logits/chosen": -3.066415309906006,
      "logits/rejected": -3.0779404640197754,
      "logps/chosen": -0.5206084251403809,
      "logps/rejected": -1.0640580654144287,
      "loss": 0.4196,
      "nll_loss": 0.34643101692199707,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.10412168502807617,
      "rewards/margins": 0.10868990421295166,
      "rewards/rejected": -0.21281161904335022,
      "step": 335
    },
    {
      "epoch": 1.46,
      "grad_norm": 4.5,
      "learning_rate": 2.606709810322957e-06,
      "log_odds_chosen": 1.0554051399230957,
      "log_odds_ratio": -0.3538056015968323,
      "logits/chosen": -3.086264133453369,
      "logits/rejected": -3.0979015827178955,
      "logps/chosen": -0.4939126968383789,
      "logps/rejected": -1.0222851037979126,
      "loss": 0.4479,
      "nll_loss": 0.34655773639678955,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09878254681825638,
      "rewards/margins": 0.10567446053028107,
      "rewards/rejected": -0.20445701479911804,
      "step": 340
    },
    {
      "epoch": 1.48,
      "grad_norm": 4.5,
      "learning_rate": 2.550558666569279e-06,
      "log_odds_chosen": 1.0255191326141357,
      "log_odds_ratio": -0.3848777115345001,
      "logits/chosen": -3.0852413177490234,
      "logits/rejected": -3.0870513916015625,
      "logps/chosen": -0.49635806679725647,
      "logps/rejected": -0.99615478515625,
      "loss": 0.4092,
      "nll_loss": 0.3158537745475769,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.09927161037921906,
      "rewards/margins": 0.0999593436717987,
      "rewards/rejected": -0.19923093914985657,
      "step": 345
    },
    {
      "epoch": 1.5,
      "grad_norm": 5.3125,
      "learning_rate": 2.4943819921147125e-06,
      "log_odds_chosen": 1.0540813207626343,
      "log_odds_ratio": -0.3640359342098236,
      "logits/chosen": -3.0244834423065186,
      "logits/rejected": -3.0654783248901367,
      "logps/chosen": -0.5098503828048706,
      "logps/rejected": -1.0328686237335205,
      "loss": 0.4198,
      "nll_loss": 0.38225817680358887,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10197006165981293,
      "rewards/margins": 0.10460364818572998,
      "rewards/rejected": -0.2065737247467041,
      "step": 350
    },
    {
      "epoch": 1.52,
      "grad_norm": 4.53125,
      "learning_rate": 2.438208154595703e-06,
      "log_odds_chosen": 1.0023833513259888,
      "log_odds_ratio": -0.3944670557975769,
      "logits/chosen": -3.053039312362671,
      "logits/rejected": -3.0673396587371826,
      "logps/chosen": -0.5665103197097778,
      "logps/rejected": -1.0646257400512695,
      "loss": 0.4357,
      "nll_loss": 0.39022982120513916,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.1133020743727684,
      "rewards/margins": 0.09962306171655655,
      "rewards/rejected": -0.21292512118816376,
      "step": 355
    },
    {
      "epoch": 1.54,
      "grad_norm": 4.40625,
      "learning_rate": 2.3820655202161237e-06,
      "log_odds_chosen": 0.9806038737297058,
      "log_odds_ratio": -0.3737329840660095,
      "logits/chosen": -3.0929837226867676,
      "logits/rejected": -3.113180637359619,
      "logps/chosen": -0.5550697445869446,
      "logps/rejected": -1.0932964086532593,
      "loss": 0.3961,
      "nll_loss": 0.35079020261764526,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.11101396381855011,
      "rewards/margins": 0.10764533281326294,
      "rewards/rejected": -0.21865928173065186,
      "step": 360
    },
    {
      "epoch": 1.56,
      "grad_norm": 4.625,
      "learning_rate": 2.3259824394231427e-06,
      "log_odds_chosen": 1.031070351600647,
      "log_odds_ratio": -0.36041268706321716,
      "logits/chosen": -3.0083768367767334,
      "logits/rejected": -3.0171380043029785,
      "logps/chosen": -0.46207037568092346,
      "logps/rejected": -0.9668736457824707,
      "loss": 0.4175,
      "nll_loss": 0.3523089289665222,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09241408109664917,
      "rewards/margins": 0.10096064954996109,
      "rewards/rejected": -0.19337472319602966,
      "step": 365
    },
    {
      "epoch": 1.58,
      "grad_norm": 4.75,
      "learning_rate": 2.2699872325910458e-06,
      "log_odds_chosen": 1.117046594619751,
      "log_odds_ratio": -0.37522634863853455,
      "logits/chosen": -3.033770799636841,
      "logits/rejected": -3.018136501312256,
      "logps/chosen": -0.5326264500617981,
      "logps/rejected": -1.1620831489562988,
      "loss": 0.3983,
      "nll_loss": 0.3253507912158966,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.10652528703212738,
      "rewards/margins": 0.12589131295681,
      "rewards/rejected": -0.23241659998893738,
      "step": 370
    },
    {
      "epoch": 1.61,
      "grad_norm": 5.125,
      "learning_rate": 2.214108175720246e-06,
      "log_odds_chosen": 0.949042022228241,
      "log_odds_ratio": -0.37258926033973694,
      "logits/chosen": -3.006155490875244,
      "logits/rejected": -3.0606319904327393,
      "logps/chosen": -0.4789462685585022,
      "logps/rejected": -0.9343479871749878,
      "loss": 0.4191,
      "nll_loss": 0.3158559799194336,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.09578924626111984,
      "rewards/margins": 0.09108035266399384,
      "rewards/rejected": -0.18686962127685547,
      "step": 375
    },
    {
      "epoch": 1.63,
      "grad_norm": 5.03125,
      "learning_rate": 2.158373486158703e-06,
      "log_odds_chosen": 0.9779850244522095,
      "log_odds_ratio": -0.3736100196838379,
      "logits/chosen": -3.0543103218078613,
      "logits/rejected": -3.0758256912231445,
      "logps/chosen": -0.46442002058029175,
      "logps/rejected": -0.9138116836547852,
      "loss": 0.4153,
      "nll_loss": 0.33239707350730896,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.09288401901721954,
      "rewards/margins": 0.08987830579280853,
      "rewards/rejected": -0.18276233971118927,
      "step": 380
    },
    {
      "epoch": 1.65,
      "grad_norm": 4.90625,
      "learning_rate": 2.1028113083529543e-06,
      "log_odds_chosen": 1.1226699352264404,
      "log_odds_ratio": -0.34678876399993896,
      "logits/chosen": -3.077775478363037,
      "logits/rejected": -3.0885062217712402,
      "logps/chosen": -0.4337676465511322,
      "logps/rejected": -0.9120414853096008,
      "loss": 0.4173,
      "nll_loss": 0.3161032497882843,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.08675353974103928,
      "rewards/margins": 0.09565478563308716,
      "rewards/rejected": -0.18240833282470703,
      "step": 385
    },
    {
      "epoch": 1.67,
      "grad_norm": 4.78125,
      "learning_rate": 2.0474496996359676e-06,
      "log_odds_chosen": 1.1200788021087646,
      "log_odds_ratio": -0.3556548058986664,
      "logits/chosen": -3.057023525238037,
      "logits/rejected": -3.0625433921813965,
      "logps/chosen": -0.47518497705459595,
      "logps/rejected": -1.0216314792633057,
      "loss": 0.4278,
      "nll_loss": 0.35161006450653076,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09503699094057083,
      "rewards/margins": 0.10928928852081299,
      "rewards/rejected": -0.20432627201080322,
      "step": 390
    },
    {
      "epoch": 1.69,
      "grad_norm": 4.8125,
      "learning_rate": 1.992316616058974e-06,
      "log_odds_chosen": 0.982783317565918,
      "log_odds_ratio": -0.3863145709037781,
      "logits/chosen": -3.0365030765533447,
      "logits/rejected": -3.0551135540008545,
      "logps/chosen": -0.5143071413040161,
      "logps/rejected": -0.9792839288711548,
      "loss": 0.4238,
      "nll_loss": 0.32200440764427185,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.1028614267706871,
      "rewards/margins": 0.09299537539482117,
      "rewards/rejected": -0.19585679471492767,
      "step": 395
    },
    {
      "epoch": 1.71,
      "grad_norm": 4.65625,
      "learning_rate": 1.937439898274455e-06,
      "log_odds_chosen": 1.0216920375823975,
      "log_odds_ratio": -0.36191970109939575,
      "logits/chosen": -3.094931125640869,
      "logits/rejected": -3.1032814979553223,
      "logps/chosen": -0.4898260235786438,
      "logps/rejected": -0.9827303886413574,
      "loss": 0.4067,
      "nll_loss": 0.33159953355789185,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09796520322561264,
      "rewards/margins": 0.09858085960149765,
      "rewards/rejected": -0.1965460628271103,
      "step": 400
    },
    {
      "epoch": 1.73,
      "grad_norm": 5.4375,
      "learning_rate": 1.882847257477398e-06,
      "log_odds_chosen": 0.9538644552230835,
      "log_odds_ratio": -0.3876429796218872,
      "logits/chosen": -3.086061954498291,
      "logits/rejected": -3.0957791805267334,
      "logps/chosen": -0.5137572288513184,
      "logps/rejected": -0.9913498759269714,
      "loss": 0.4317,
      "nll_loss": 0.34138768911361694,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10275144875049591,
      "rewards/margins": 0.09551852941513062,
      "rewards/rejected": -0.19826997816562653,
      "step": 405
    },
    {
      "epoch": 1.76,
      "grad_norm": 4.25,
      "learning_rate": 1.8285662614119185e-06,
      "log_odds_chosen": 1.1262160539627075,
      "log_odds_ratio": -0.36318790912628174,
      "logits/chosen": -3.102365016937256,
      "logits/rejected": -3.0945730209350586,
      "logps/chosen": -0.5087975859642029,
      "logps/rejected": -1.0988045930862427,
      "loss": 0.3526,
      "nll_loss": 0.2773934006690979,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.10175951570272446,
      "rewards/margins": 0.11800140142440796,
      "rewards/rejected": -0.2197609394788742,
      "step": 410
    },
    {
      "epoch": 1.78,
      "grad_norm": 4.6875,
      "learning_rate": 1.7746243204503342e-06,
      "log_odds_chosen": 1.0629944801330566,
      "log_odds_ratio": -0.35559511184692383,
      "logits/chosen": -3.082392692565918,
      "logits/rejected": -3.081556558609009,
      "logps/chosen": -0.5505409240722656,
      "logps/rejected": -1.1507011651992798,
      "loss": 0.4123,
      "nll_loss": 0.35343068838119507,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.11010818183422089,
      "rewards/margins": 0.12003204971551895,
      "rewards/rejected": -0.23014025390148163,
      "step": 415
    },
    {
      "epoch": 1.8,
      "grad_norm": 4.96875,
      "learning_rate": 1.7210486737516947e-06,
      "log_odds_chosen": 1.1333677768707275,
      "log_odds_ratio": -0.3568989932537079,
      "logits/chosen": -3.0343739986419678,
      "logits/rejected": -3.0451571941375732,
      "logps/chosen": -0.5825051069259644,
      "logps/rejected": -1.2207900285720825,
      "loss": 0.4581,
      "nll_loss": 0.43837055563926697,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.11650101095438004,
      "rewards/margins": 0.12765701115131378,
      "rewards/rejected": -0.24415802955627441,
      "step": 420
    },
    {
      "epoch": 1.82,
      "grad_norm": 4.75,
      "learning_rate": 1.6678663755067765e-06,
      "log_odds_chosen": 1.1494513750076294,
      "log_odds_ratio": -0.34156522154808044,
      "logits/chosen": -3.067613124847412,
      "logits/rejected": -3.0813510417938232,
      "logps/chosen": -0.5272513628005981,
      "logps/rejected": -1.1254432201385498,
      "loss": 0.4457,
      "nll_loss": 0.3762899935245514,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10545027256011963,
      "rewards/margins": 0.1196383684873581,
      "rewards/rejected": -0.22508864104747772,
      "step": 425
    },
    {
      "epoch": 1.84,
      "grad_norm": 5.3125,
      "learning_rate": 1.6151042812764798e-06,
      "log_odds_chosen": 1.0926806926727295,
      "log_odds_ratio": -0.35124513506889343,
      "logits/chosen": -3.1003775596618652,
      "logits/rejected": -3.0869221687316895,
      "logps/chosen": -0.5167126059532166,
      "logps/rejected": -1.0946365594863892,
      "loss": 0.4301,
      "nll_loss": 0.3912351429462433,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.10334251821041107,
      "rewards/margins": 0.11558476835489273,
      "rewards/rejected": -0.2189272940158844,
      "step": 430
    },
    {
      "epoch": 1.86,
      "grad_norm": 5.40625,
      "learning_rate": 1.5627890344305256e-06,
      "log_odds_chosen": 1.0168402194976807,
      "log_odds_ratio": -0.37472400069236755,
      "logits/chosen": -3.073152542114258,
      "logits/rejected": -3.062530279159546,
      "logps/chosen": -0.4963172376155853,
      "logps/rejected": -0.9966226816177368,
      "loss": 0.4033,
      "nll_loss": 0.3146648108959198,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09926344454288483,
      "rewards/margins": 0.10006110370159149,
      "rewards/rejected": -0.1993245631456375,
      "step": 435
    },
    {
      "epoch": 1.88,
      "grad_norm": 4.375,
      "learning_rate": 1.5109470526933083e-06,
      "log_odds_chosen": 1.3515523672103882,
      "log_odds_ratio": -0.2944795489311218,
      "logits/chosen": -3.0687355995178223,
      "logits/rejected": -3.062213897705078,
      "logps/chosen": -0.45598679780960083,
      "logps/rejected": -1.140779733657837,
      "loss": 0.4087,
      "nll_loss": 0.32488110661506653,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09119735658168793,
      "rewards/margins": 0.13695858418941498,
      "rewards/rejected": -0.2281559407711029,
      "step": 440
    },
    {
      "epoch": 1.91,
      "grad_norm": 4.75,
      "learning_rate": 1.4596045148036878e-06,
      "log_odds_chosen": 1.054136037826538,
      "log_odds_ratio": -0.3551008701324463,
      "logits/chosen": -3.0608222484588623,
      "logits/rejected": -3.068776845932007,
      "logps/chosen": -0.4831467270851135,
      "logps/rejected": -1.002421259880066,
      "loss": 0.4281,
      "nll_loss": 0.39750441908836365,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.09662934392690659,
      "rewards/margins": 0.10385493189096451,
      "rewards/rejected": -0.2004842460155487,
      "step": 445
    },
    {
      "epoch": 1.93,
      "grad_norm": 4.9375,
      "learning_rate": 1.4087873472954638e-06,
      "log_odds_chosen": 1.0801405906677246,
      "log_odds_ratio": -0.3470378816127777,
      "logits/chosen": -3.1068406105041504,
      "logits/rejected": -3.1202595233917236,
      "logps/chosen": -0.4985682964324951,
      "logps/rejected": -1.0175180435180664,
      "loss": 0.3913,
      "nll_loss": 0.3334752023220062,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09971366077661514,
      "rewards/margins": 0.10378994792699814,
      "rewards/rejected": -0.20350360870361328,
      "step": 450
    },
    {
      "epoch": 1.95,
      "grad_norm": 4.53125,
      "learning_rate": 1.3585212114052104e-06,
      "log_odds_chosen": 0.9953893423080444,
      "log_odds_ratio": -0.37009868025779724,
      "logits/chosen": -3.096047878265381,
      "logits/rejected": -3.1102185249328613,
      "logps/chosen": -0.5521574020385742,
      "logps/rejected": -1.0702391862869263,
      "loss": 0.4248,
      "nll_loss": 0.3823598027229309,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.1104314923286438,
      "rewards/margins": 0.10361631959676743,
      "rewards/rejected": -0.21404781937599182,
      "step": 455
    },
    {
      "epoch": 1.97,
      "grad_norm": 4.90625,
      "learning_rate": 1.308831490114072e-06,
      "log_odds_chosen": 1.0446946620941162,
      "log_odds_ratio": -0.3528267443180084,
      "logits/chosen": -3.1083786487579346,
      "logits/rejected": -3.115349531173706,
      "logps/chosen": -0.4900852143764496,
      "logps/rejected": -1.0257424116134644,
      "loss": 0.4454,
      "nll_loss": 0.30312374234199524,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.09801704436540604,
      "rewards/margins": 0.10713144391775131,
      "rewards/rejected": -0.20514848828315735,
      "step": 460
    },
    {
      "epoch": 1.99,
      "grad_norm": 4.71875,
      "learning_rate": 1.2597432753300753e-06,
      "log_odds_chosen": 1.0419663190841675,
      "log_odds_ratio": -0.3548406660556793,
      "logits/chosen": -3.0856945514678955,
      "logits/rejected": -3.095820188522339,
      "logps/chosen": -0.5246998071670532,
      "logps/rejected": -1.0223771333694458,
      "loss": 0.4401,
      "nll_loss": 0.44383564591407776,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.10493995994329453,
      "rewards/margins": 0.09953545033931732,
      "rewards/rejected": -0.20447543263435364,
      "step": 465
    },
    {
      "epoch": 2.01,
      "grad_norm": 5.1875,
      "learning_rate": 1.211281355217424e-06,
      "log_odds_chosen": 1.294676423072815,
      "log_odds_ratio": -0.3114815652370453,
      "logits/chosen": -3.0750784873962402,
      "logits/rejected": -3.051118850708008,
      "logps/chosen": -0.41240859031677246,
      "logps/rejected": -1.030005693435669,
      "loss": 0.3968,
      "nll_loss": 0.33392661809921265,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.08248171955347061,
      "rewards/margins": 0.1235194206237793,
      "rewards/rejected": -0.2060011327266693,
      "step": 470
    },
    {
      "epoch": 2.03,
      "grad_norm": 4.1875,
      "learning_rate": 1.1634702016791722e-06,
      "log_odds_chosen": 1.320809245109558,
      "log_odds_ratio": -0.3077535033226013,
      "logits/chosen": -3.092358350753784,
      "logits/rejected": -3.0973782539367676,
      "logps/chosen": -0.47875672578811646,
      "logps/rejected": -1.1592457294464111,
      "loss": 0.3882,
      "nll_loss": 0.3666633367538452,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.09575134515762329,
      "rewards/margins": 0.13609783351421356,
      "rewards/rejected": -0.23184919357299805,
      "step": 475
    },
    {
      "epoch": 2.06,
      "grad_norm": 4.75,
      "learning_rate": 1.116333957999608e-06,
      "log_odds_chosen": 1.5810959339141846,
      "log_odds_ratio": -0.253048837184906,
      "logits/chosen": -3.034074068069458,
      "logits/rejected": -3.0341789722442627,
      "logps/chosen": -0.40957292914390564,
      "logps/rejected": -1.1751987934112549,
      "loss": 0.3951,
      "nll_loss": 0.3588045537471771,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08191458880901337,
      "rewards/margins": 0.15312522649765015,
      "rewards/rejected": -0.23503978550434113,
      "step": 480
    },
    {
      "epoch": 2.08,
      "grad_norm": 4.5,
      "learning_rate": 1.0698964266525706e-06,
      "log_odds_chosen": 1.760838508605957,
      "log_odds_ratio": -0.23076216876506805,
      "logits/chosen": -3.011136531829834,
      "logits/rejected": -3.034350872039795,
      "logps/chosen": -0.3941037952899933,
      "logps/rejected": -1.2701072692871094,
      "loss": 0.3596,
      "nll_loss": 0.3343369960784912,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.07882075756788254,
      "rewards/margins": 0.17520073056221008,
      "rewards/rejected": -0.25402146577835083,
      "step": 485
    },
    {
      "epoch": 2.1,
      "grad_norm": 4.5625,
      "learning_rate": 1.024181057281875e-06,
      "log_odds_chosen": 1.696458101272583,
      "log_odds_ratio": -0.22928357124328613,
      "logits/chosen": -3.036642551422119,
      "logits/rejected": -3.054276943206787,
      "logps/chosen": -0.4274802803993225,
      "logps/rejected": -1.2700543403625488,
      "loss": 0.3901,
      "nll_loss": 0.3817359507083893,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.08549605309963226,
      "rewards/margins": 0.16851481795310974,
      "rewards/rejected": -0.2540109157562256,
      "step": 490
    },
    {
      "epoch": 2.12,
      "grad_norm": 4.09375,
      "learning_rate": 9.792109348599036e-07,
      "log_odds_chosen": 1.7724546194076538,
      "log_odds_ratio": -0.22515392303466797,
      "logits/chosen": -3.0674076080322266,
      "logits/rejected": -3.0711779594421387,
      "logps/chosen": -0.3904511034488678,
      "logps/rejected": -1.2670060396194458,
      "loss": 0.3537,
      "nll_loss": 0.31840625405311584,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.07809022068977356,
      "rewards/margins": 0.17531099915504456,
      "rewards/rejected": -0.2534012198448181,
      "step": 495
    },
    {
      "epoch": 2.14,
      "grad_norm": 4.0625,
      "learning_rate": 9.350087680303441e-07,
      "log_odds_chosen": 1.543501615524292,
      "log_odds_ratio": -0.27717000246047974,
      "logits/chosen": -3.0165324211120605,
      "logits/rejected": -3.0413506031036377,
      "logps/chosen": -0.4150218367576599,
      "logps/rejected": -1.1760425567626953,
      "loss": 0.347,
      "nll_loss": 0.30745163559913635,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.08300437033176422,
      "rewards/margins": 0.15220412611961365,
      "rewards/rejected": -0.23520851135253906,
      "step": 500
    },
    {
      "epoch": 2.16,
      "grad_norm": 4.28125,
      "learning_rate": 8.915968776409673e-07,
      "log_odds_chosen": 1.6620781421661377,
      "log_odds_ratio": -0.2480989396572113,
      "logits/chosen": -2.983677387237549,
      "logits/rejected": -3.0116188526153564,
      "logps/chosen": -0.41676464676856995,
      "logps/rejected": -1.2172657251358032,
      "loss": 0.383,
      "nll_loss": 0.37900644540786743,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08335293084383011,
      "rewards/margins": 0.16010019183158875,
      "rewards/rejected": -0.24345311522483826,
      "step": 505
    },
    {
      "epoch": 2.18,
      "grad_norm": 4.09375,
      "learning_rate": 8.48997185472226e-07,
      "log_odds_chosen": 1.6592066287994385,
      "log_odds_ratio": -0.24125425517559052,
      "logits/chosen": -3.0089008808135986,
      "logits/rejected": -3.0470426082611084,
      "logps/chosen": -0.3994481563568115,
      "logps/rejected": -1.2132692337036133,
      "loss": 0.3553,
      "nll_loss": 0.310027539730072,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07988963276147842,
      "rewards/margins": 0.16276419162750244,
      "rewards/rejected": -0.24265384674072266,
      "step": 510
    },
    {
      "epoch": 2.21,
      "grad_norm": 4.3125,
      "learning_rate": 8.072312031673757e-07,
      "log_odds_chosen": 1.6442210674285889,
      "log_odds_ratio": -0.25921303033828735,
      "logits/chosen": -3.0381431579589844,
      "logits/rejected": -3.039233446121216,
      "logps/chosen": -0.4131619334220886,
      "logps/rejected": -1.2261427640914917,
      "loss": 0.3536,
      "nll_loss": 0.26558876037597656,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.0826324000954628,
      "rewards/margins": 0.16259616613388062,
      "rewards/rejected": -0.245228573679924,
      "step": 515
    },
    {
      "epoch": 2.23,
      "grad_norm": 4.15625,
      "learning_rate": 7.663200213697012e-07,
      "log_odds_chosen": 1.571980595588684,
      "log_odds_ratio": -0.25617414712905884,
      "logits/chosen": -3.0464415550231934,
      "logits/rejected": -3.056257724761963,
      "logps/chosen": -0.45917248725891113,
      "logps/rejected": -1.2653144598007202,
      "loss": 0.3709,
      "nll_loss": 0.2762446701526642,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.09183450043201447,
      "rewards/margins": 0.16122838854789734,
      "rewards/rejected": -0.253062903881073,
      "step": 520
    },
    {
      "epoch": 2.25,
      "grad_norm": 3.734375,
      "learning_rate": 7.26284299072334e-07,
      "log_odds_chosen": 1.5702177286148071,
      "log_odds_ratio": -0.25459572672843933,
      "logits/chosen": -3.0374367237091064,
      "logits/rejected": -3.0311105251312256,
      "logps/chosen": -0.3985027074813843,
      "logps/rejected": -1.1368948221206665,
      "loss": 0.3357,
      "nll_loss": 0.3179529309272766,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.0797005444765091,
      "rewards/margins": 0.147678405046463,
      "rewards/rejected": -0.22737888991832733,
      "step": 525
    },
    {
      "epoch": 2.27,
      "grad_norm": 4.125,
      "learning_rate": 6.871442531860495e-07,
      "log_odds_chosen": 1.7363159656524658,
      "log_odds_ratio": -0.21836872398853302,
      "logits/chosen": -3.0608057975769043,
      "logits/rejected": -3.075495958328247,
      "logps/chosen": -0.4046151638031006,
      "logps/rejected": -1.2606099843978882,
      "loss": 0.3318,
      "nll_loss": 0.2710905075073242,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08092303574085236,
      "rewards/margins": 0.1711989939212799,
      "rewards/rejected": -0.2521219849586487,
      "step": 530
    },
    {
      "epoch": 2.29,
      "grad_norm": 4.3125,
      "learning_rate": 6.489196483302895e-07,
      "log_odds_chosen": 1.8925565481185913,
      "log_odds_ratio": -0.23906934261322021,
      "logits/chosen": -3.055502414703369,
      "logits/rejected": -3.0740723609924316,
      "logps/chosen": -0.3803566098213196,
      "logps/rejected": -1.2806613445281982,
      "loss": 0.3988,
      "nll_loss": 0.34115779399871826,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07607132941484451,
      "rewards/margins": 0.18006090819835663,
      "rewards/rejected": -0.25613224506378174,
      "step": 535
    },
    {
      "epoch": 2.31,
      "grad_norm": 5.03125,
      "learning_rate": 6.11629786852592e-07,
      "log_odds_chosen": 1.6516625881195068,
      "log_odds_ratio": -0.22780461609363556,
      "logits/chosen": -3.0414628982543945,
      "logits/rejected": -3.0669968128204346,
      "logps/chosen": -0.45682182908058167,
      "logps/rejected": -1.346400260925293,
      "loss": 0.3627,
      "nll_loss": 0.31683245301246643,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09136436879634857,
      "rewards/margins": 0.17791566252708435,
      "rewards/rejected": -0.2692800462245941,
      "step": 540
    },
    {
      "epoch": 2.33,
      "grad_norm": 4.15625,
      "learning_rate": 5.752934990814477e-07,
      "log_odds_chosen": 1.6729217767715454,
      "log_odds_ratio": -0.2464570254087448,
      "logits/chosen": -3.0506749153137207,
      "logits/rejected": -3.053323268890381,
      "logps/chosen": -0.42784634232521057,
      "logps/rejected": -1.2647467851638794,
      "loss": 0.3509,
      "nll_loss": 0.28274843096733093,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.08556927740573883,
      "rewards/margins": 0.16738007962703705,
      "rewards/rejected": -0.2529493570327759,
      "step": 545
    },
    {
      "epoch": 2.36,
      "grad_norm": 4.40625,
      "learning_rate": 5.399291338175158e-07,
      "log_odds_chosen": 1.715243935585022,
      "log_odds_ratio": -0.2278958261013031,
      "logits/chosen": -3.029287338256836,
      "logits/rejected": -3.0767955780029297,
      "logps/chosen": -0.3604409694671631,
      "logps/rejected": -1.1182682514190674,
      "loss": 0.3488,
      "nll_loss": 0.2922760248184204,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.07208819687366486,
      "rewards/margins": 0.15156546235084534,
      "rewards/rejected": -0.223653644323349,
      "step": 550
    },
    {
      "epoch": 2.38,
      "grad_norm": 4.46875,
      "learning_rate": 5.055545490679981e-07,
      "log_odds_chosen": 1.8542802333831787,
      "log_odds_ratio": -0.23659536242485046,
      "logits/chosen": -3.0523695945739746,
      "logits/rejected": -3.0597970485687256,
      "logps/chosen": -0.45028233528137207,
      "logps/rejected": -1.422621488571167,
      "loss": 0.3544,
      "nll_loss": 0.29960012435913086,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09005647152662277,
      "rewards/margins": 0.19446782767772675,
      "rewards/rejected": -0.2845243215560913,
      "step": 555
    },
    {
      "epoch": 2.4,
      "grad_norm": 4.5,
      "learning_rate": 4.721871030288458e-07,
      "log_odds_chosen": 1.7230913639068604,
      "log_odds_ratio": -0.22683128714561462,
      "logits/chosen": -3.0413856506347656,
      "logits/rejected": -3.048562526702881,
      "logps/chosen": -0.388246089220047,
      "logps/rejected": -1.230250358581543,
      "loss": 0.3764,
      "nll_loss": 0.2896038293838501,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07764922082424164,
      "rewards/margins": 0.1684008538722992,
      "rewards/rejected": -0.24605008959770203,
      "step": 560
    },
    {
      "epoch": 2.42,
      "grad_norm": 4.125,
      "learning_rate": 4.3984364531936497e-07,
      "log_odds_chosen": 1.764896035194397,
      "log_odds_ratio": -0.2130444496870041,
      "logits/chosen": -2.9862923622131348,
      "logits/rejected": -3.0289909839630127,
      "logps/chosen": -0.431601345539093,
      "logps/rejected": -1.3558611869812012,
      "loss": 0.3606,
      "nll_loss": 0.31346598267555237,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08632026612758636,
      "rewards/margins": 0.1848519742488861,
      "rewards/rejected": -0.2711722254753113,
      "step": 565
    },
    {
      "epoch": 2.44,
      "grad_norm": 4.25,
      "learning_rate": 4.0854050847362966e-07,
      "log_odds_chosen": 1.7146213054656982,
      "log_odds_ratio": -0.22591781616210938,
      "logits/chosen": -3.002671480178833,
      "logits/rejected": -3.0256471633911133,
      "logps/chosen": -0.42149290442466736,
      "logps/rejected": -1.249757170677185,
      "loss": 0.349,
      "nll_loss": 0.28486207127571106,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.08429859578609467,
      "rewards/margins": 0.16565285623073578,
      "rewards/rejected": -0.24995143711566925,
      "step": 570
    },
    {
      "epoch": 2.46,
      "grad_norm": 3.953125,
      "learning_rate": 3.782934996930135e-07,
      "log_odds_chosen": 1.5356372594833374,
      "log_odds_ratio": -0.24702449142932892,
      "logits/chosen": -3.0242128372192383,
      "logits/rejected": -3.0081803798675537,
      "logps/chosen": -0.4465559124946594,
      "logps/rejected": -1.2361305952072144,
      "loss": 0.3485,
      "nll_loss": 0.28072112798690796,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.08931118249893188,
      "rewards/margins": 0.15791496634483337,
      "rewards/rejected": -0.24722616374492645,
      "step": 575
    },
    {
      "epoch": 2.48,
      "grad_norm": 5.09375,
      "learning_rate": 3.49117892864001e-07,
      "log_odds_chosen": 1.6950727701187134,
      "log_odds_ratio": -0.2591921091079712,
      "logits/chosen": -3.076004981994629,
      "logits/rejected": -3.0650579929351807,
      "logps/chosen": -0.4506012499332428,
      "logps/rejected": -1.3095630407333374,
      "loss": 0.3732,
      "nll_loss": 0.33324551582336426,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.09012024849653244,
      "rewards/margins": 0.17179235816001892,
      "rewards/rejected": -0.26191261410713196,
      "step": 580
    },
    {
      "epoch": 2.51,
      "grad_norm": 4.59375,
      "learning_rate": 3.2102842084530293e-07,
      "log_odds_chosen": 1.7578738927841187,
      "log_odds_ratio": -0.2337111532688141,
      "logits/chosen": -3.021003246307373,
      "logits/rejected": -3.0427439212799072,
      "logps/chosen": -0.4114065170288086,
      "logps/rejected": -1.3049488067626953,
      "loss": 0.3608,
      "nll_loss": 0.3577117323875427,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08228130638599396,
      "rewards/margins": 0.17870847880840302,
      "rewards/rejected": -0.26098981499671936,
      "step": 585
    },
    {
      "epoch": 2.53,
      "grad_norm": 4.4375,
      "learning_rate": 2.940392680281798e-07,
      "log_odds_chosen": 1.8219060897827148,
      "log_odds_ratio": -0.22410018742084503,
      "logits/chosen": -3.0648932456970215,
      "logits/rejected": -3.0744993686676025,
      "logps/chosen": -0.43264156579971313,
      "logps/rejected": -1.3604795932769775,
      "loss": 0.3818,
      "nll_loss": 0.3207879960536957,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.08652831614017487,
      "rewards/margins": 0.18556760251522064,
      "rewards/rejected": -0.2720959484577179,
      "step": 590
    },
    {
      "epoch": 2.55,
      "grad_norm": 4.3125,
      "learning_rate": 2.681640631737245e-07,
      "log_odds_chosen": 1.6481034755706787,
      "log_odds_ratio": -0.25634631514549255,
      "logits/chosen": -2.9935100078582764,
      "logits/rejected": -2.9783313274383545,
      "logps/chosen": -0.4389920234680176,
      "logps/rejected": -1.241737961769104,
      "loss": 0.3654,
      "nll_loss": 0.3387255072593689,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08779840171337128,
      "rewards/margins": 0.16054919362068176,
      "rewards/rejected": -0.24834756553173065,
      "step": 595
    },
    {
      "epoch": 2.57,
      "grad_norm": 4.90625,
      "learning_rate": 2.4341587253072035e-07,
      "log_odds_chosen": 1.658064842224121,
      "log_odds_ratio": -0.23493357002735138,
      "logits/chosen": -3.0567028522491455,
      "logits/rejected": -3.068887233734131,
      "logps/chosen": -0.41281405091285706,
      "logps/rejected": -1.230850338935852,
      "loss": 0.3731,
      "nll_loss": 0.31240588426589966,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08256281167268753,
      "rewards/margins": 0.16360723972320557,
      "rewards/rejected": -0.2461700737476349,
      "step": 600
    },
    {
      "epoch": 2.59,
      "grad_norm": 3.8125,
      "learning_rate": 2.1980719323755533e-07,
      "log_odds_chosen": 1.6675655841827393,
      "log_odds_ratio": -0.24681706726551056,
      "logits/chosen": -3.039940595626831,
      "logits/rejected": -3.0746922492980957,
      "logps/chosen": -0.44124722480773926,
      "logps/rejected": -1.2557570934295654,
      "loss": 0.375,
      "nll_loss": 0.31266745924949646,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08824944496154785,
      "rewards/margins": 0.16290199756622314,
      "rewards/rejected": -0.2511514127254486,
      "step": 605
    },
    {
      "epoch": 2.61,
      "grad_norm": 4.90625,
      "learning_rate": 1.9734994701152004e-07,
      "log_odds_chosen": 1.6424992084503174,
      "log_odds_ratio": -0.25031501054763794,
      "logits/chosen": -3.055297374725342,
      "logits/rejected": -3.062410831451416,
      "logps/chosen": -0.42411407828330994,
      "logps/rejected": -1.2362747192382812,
      "loss": 0.3487,
      "nll_loss": 0.30518966913223267,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08482281118631363,
      "rewards/margins": 0.16243210434913635,
      "rewards/rejected": -0.24725492298603058,
      "step": 610
    },
    {
      "epoch": 2.63,
      "grad_norm": 5.0,
      "learning_rate": 1.7605547412867574e-07,
      "log_odds_chosen": 2.0515856742858887,
      "log_odds_ratio": -0.209257572889328,
      "logits/chosen": -3.0167908668518066,
      "logits/rejected": -3.015455484390259,
      "logps/chosen": -0.36906909942626953,
      "logps/rejected": -1.372367262840271,
      "loss": 0.3962,
      "nll_loss": 0.3273320496082306,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07381383329629898,
      "rewards/margins": 0.20065966248512268,
      "rewards/rejected": -0.27447348833084106,
      "step": 615
    },
    {
      "epoch": 2.66,
      "grad_norm": 4.5,
      "learning_rate": 1.5593452769733425e-07,
      "log_odds_chosen": 1.8267831802368164,
      "log_odds_ratio": -0.21358974277973175,
      "logits/chosen": -3.0070767402648926,
      "logits/rejected": -3.028520107269287,
      "logps/chosen": -0.42965587973594666,
      "logps/rejected": -1.3311954736709595,
      "loss": 0.3591,
      "nll_loss": 0.3173789978027344,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.08593117445707321,
      "rewards/margins": 0.18030789494514465,
      "rewards/rejected": -0.26623910665512085,
      "step": 620
    },
    {
      "epoch": 2.68,
      "grad_norm": 4.625,
      "learning_rate": 1.369972682280435e-07,
      "log_odds_chosen": 1.6776520013809204,
      "log_odds_ratio": -0.24110755324363708,
      "logits/chosen": -3.0178725719451904,
      "logits/rejected": -3.041224956512451,
      "logps/chosen": -0.4312921166419983,
      "logps/rejected": -1.2802475690841675,
      "loss": 0.358,
      "nll_loss": 0.2894738018512726,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.0862584188580513,
      "rewards/margins": 0.169791117310524,
      "rewards/rejected": -0.2560495436191559,
      "step": 625
    },
    {
      "epoch": 2.7,
      "grad_norm": 4.15625,
      "learning_rate": 1.1925325850281416e-07,
      "log_odds_chosen": 1.7899662256240845,
      "log_odds_ratio": -0.22585618495941162,
      "logits/chosen": -3.020197868347168,
      "logits/rejected": -3.054832935333252,
      "logps/chosen": -0.38960325717926025,
      "logps/rejected": -1.2320774793624878,
      "loss": 0.3746,
      "nll_loss": 0.31026405096054077,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.07792064547538757,
      "rewards/margins": 0.16849488019943237,
      "rewards/rejected": -0.24641552567481995,
      "step": 630
    },
    {
      "epoch": 2.72,
      "grad_norm": 4.625,
      "learning_rate": 1.0271145874618526e-07,
      "log_odds_chosen": 1.7390544414520264,
      "log_odds_ratio": -0.20484574139118195,
      "logits/chosen": -3.0023467540740967,
      "logits/rejected": -3.039856433868408,
      "logps/chosen": -0.45310506224632263,
      "logps/rejected": -1.3705298900604248,
      "loss": 0.3542,
      "nll_loss": 0.32287827134132385,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -0.09062100946903229,
      "rewards/margins": 0.1834850013256073,
      "rewards/rejected": -0.274105966091156,
      "step": 635
    },
    {
      "epoch": 2.74,
      "grad_norm": 3.921875,
      "learning_rate": 8.738022210056318e-08,
      "log_odds_chosen": 1.708214521408081,
      "log_odds_ratio": -0.2522994875907898,
      "logits/chosen": -3.050048828125,
      "logits/rejected": -3.041829824447632,
      "logps/chosen": -0.38343745470046997,
      "logps/rejected": -1.1687476634979248,
      "loss": 0.3592,
      "nll_loss": 0.3253324627876282,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.07668749988079071,
      "rewards/margins": 0.15706202387809753,
      "rewards/rejected": -0.23374950885772705,
      "step": 640
    },
    {
      "epoch": 2.76,
      "grad_norm": 4.34375,
      "learning_rate": 7.326729040812136e-08,
      "log_odds_chosen": 1.5645431280136108,
      "log_odds_ratio": -0.2705634534358978,
      "logits/chosen": -3.0365240573883057,
      "logits/rejected": -3.0504961013793945,
      "logps/chosen": -0.4279886782169342,
      "logps/rejected": -1.212249517440796,
      "loss": 0.3985,
      "nll_loss": 0.3430742025375366,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08559773117303848,
      "rewards/margins": 0.15685215592384338,
      "rewards/rejected": -0.24244992434978485,
      "step": 645
    },
    {
      "epoch": 2.78,
      "grad_norm": 4.71875,
      "learning_rate": 6.037979030138697e-08,
      "log_odds_chosen": 1.7532436847686768,
      "log_odds_ratio": -0.23098623752593994,
      "logits/chosen": -3.0512053966522217,
      "logits/rejected": -3.0449166297912598,
      "logps/chosen": -0.41148900985717773,
      "logps/rejected": -1.261879324913025,
      "loss": 0.3527,
      "nll_loss": 0.32637524604797363,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.08229780942201614,
      "rewards/margins": 0.1700780689716339,
      "rewards/rejected": -0.25237587094306946,
      "step": 650
    },
    {
      "epoch": 2.81,
      "grad_norm": 4.71875,
      "learning_rate": 4.872422960449264e-08,
      "log_odds_chosen": 1.7282205820083618,
      "log_odds_ratio": -0.24718597531318665,
      "logits/chosen": -3.029437303543091,
      "logits/rejected": -3.033756732940674,
      "logps/chosen": -0.39554673433303833,
      "logps/rejected": -1.2203553915023804,
      "loss": 0.3649,
      "nll_loss": 0.2759321331977844,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07910935580730438,
      "rewards/margins": 0.1649617701768875,
      "rewards/rejected": -0.2440710961818695,
      "step": 655
    },
    {
      "epoch": 2.83,
      "grad_norm": 4.59375,
      "learning_rate": 3.830649404690939e-08,
      "log_odds_chosen": 1.5842225551605225,
      "log_odds_ratio": -0.2688426673412323,
      "logits/chosen": -3.002917766571045,
      "logits/rejected": -3.0281500816345215,
      "logps/chosen": -0.4680345058441162,
      "logps/rejected": -1.28192937374115,
      "loss": 0.3717,
      "nll_loss": 0.33302080631256104,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.09360691159963608,
      "rewards/margins": 0.16277894377708435,
      "rewards/rejected": -0.25638583302497864,
      "step": 660
    },
    {
      "epoch": 2.85,
      "grad_norm": 4.59375,
      "learning_rate": 2.91318442913191e-08,
      "log_odds_chosen": 1.861280083656311,
      "log_odds_ratio": -0.22339418530464172,
      "logits/chosen": -3.0236425399780273,
      "logits/rejected": -3.0673043727874756,
      "logps/chosen": -0.4108535349369049,
      "logps/rejected": -1.3879892826080322,
      "loss": 0.3649,
      "nll_loss": 0.25299254059791565,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.08217070251703262,
      "rewards/margins": 0.19542714953422546,
      "rewards/rejected": -0.2775978446006775,
      "step": 665
    },
    {
      "epoch": 2.87,
      "grad_norm": 4.5625,
      "learning_rate": 2.120491327712776e-08,
      "log_odds_chosen": 1.5737533569335938,
      "log_odds_ratio": -0.28394436836242676,
      "logits/chosen": -3.027447462081909,
      "logits/rejected": -3.0334548950195312,
      "logps/chosen": -0.44802603125572205,
      "logps/rejected": -1.257824182510376,
      "loss": 0.3932,
      "nll_loss": 0.3312658965587616,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.08960519731044769,
      "rewards/margins": 0.16195961833000183,
      "rewards/rejected": -0.25156480073928833,
      "step": 670
    },
    {
      "epoch": 2.89,
      "grad_norm": 4.5625,
      "learning_rate": 1.452970388096192e-08,
      "log_odds_chosen": 1.8042373657226562,
      "log_odds_ratio": -0.21902310848236084,
      "logits/chosen": -3.0271263122558594,
      "logits/rejected": -3.0365521907806396,
      "logps/chosen": -0.39096662402153015,
      "logps/rejected": -1.293257713317871,
      "loss": 0.3928,
      "nll_loss": 0.344696581363678,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.07819332182407379,
      "rewards/margins": 0.18045823276042938,
      "rewards/rejected": -0.2586515545845032,
      "step": 675
    },
    {
      "epoch": 2.91,
      "grad_norm": 4.46875,
      "learning_rate": 9.109586895329203e-09,
      "log_odds_chosen": 1.4978595972061157,
      "log_odds_ratio": -0.24738021194934845,
      "logits/chosen": -3.0044121742248535,
      "logits/rejected": -3.0400876998901367,
      "logps/chosen": -0.3905513882637024,
      "logps/rejected": -1.0503321886062622,
      "loss": 0.3701,
      "nll_loss": 0.30373144149780273,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07811026275157928,
      "rewards/margins": 0.13195618987083435,
      "rewards/rejected": -0.21006643772125244,
      "step": 680
    },
    {
      "epoch": 2.93,
      "grad_norm": 4.5,
      "learning_rate": 4.947299326462929e-09,
      "log_odds_chosen": 1.703033208847046,
      "log_odds_ratio": -0.23556920886039734,
      "logits/chosen": -3.018275737762451,
      "logits/rejected": -3.023285388946533,
      "logps/chosen": -0.41973185539245605,
      "logps/rejected": -1.2890293598175049,
      "loss": 0.3621,
      "nll_loss": 0.34329932928085327,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.08394636958837509,
      "rewards/margins": 0.17385947704315186,
      "rewards/rejected": -0.25780585408210754,
      "step": 685
    },
    {
      "epoch": 2.96,
      "grad_norm": 4.78125,
      "learning_rate": 2.044943012210754e-09,
      "log_odds_chosen": 1.9632880687713623,
      "log_odds_ratio": -0.1982704997062683,
      "logits/chosen": -3.0093159675598145,
      "logits/rejected": -3.0189404487609863,
      "logps/chosen": -0.3720591068267822,
      "logps/rejected": -1.3600140810012817,
      "loss": 0.3384,
      "nll_loss": 0.26117128133773804,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07441182434558868,
      "rewards/margins": 0.19759100675582886,
      "rewards/rejected": -0.27200281620025635,
      "step": 690
    },
    {
      "epoch": 2.98,
      "grad_norm": 4.5,
      "learning_rate": 4.0398356066673195e-10,
      "log_odds_chosen": 1.7920351028442383,
      "log_odds_ratio": -0.21135714650154114,
      "logits/chosen": -3.0156197547912598,
      "logits/rejected": -3.0277273654937744,
      "logps/chosen": -0.41774335503578186,
      "logps/rejected": -1.3216874599456787,
      "loss": 0.3905,
      "nll_loss": 0.38513460755348206,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.08354867249727249,
      "rewards/margins": 0.1807888001203537,
      "rewards/rejected": -0.2643374800682068,
      "step": 695
    },
    {
      "epoch": 2.99,
      "step": 699,
      "total_flos": 0.0,
      "train_loss": 0.45327196680595605,
      "train_runtime": 9631.2387,
      "train_samples_per_second": 4.65,
      "train_steps_per_second": 0.073
    }
  ],
  "logging_steps": 5,
  "max_steps": 699,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}