trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 1359,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "grad_norm": 0.416015625,
      "learning_rate": 3.676470588235294e-09,
      "logits/chosen": 0.3823161721229553,
      "logits/rejected": 0.49768388271331787,
      "logps/chosen": -143.86793518066406,
      "logps/rejected": -156.11575317382812,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/margins_max": 0.0,
      "rewards/margins_min": 0.0,
      "rewards/margins_std": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.01,
      "grad_norm": 0.5,
      "learning_rate": 3.676470588235294e-08,
      "logits/chosen": -0.03673278167843819,
      "logits/rejected": 0.7153948545455933,
      "logps/chosen": -280.6231689453125,
      "logps/rejected": -194.65892028808594,
      "loss": 0.6933,
      "rewards/accuracies": 0.3888888955116272,
      "rewards/chosen": -0.0011863290565088391,
      "rewards/margins": -0.00027060293359681964,
      "rewards/margins_max": 0.0016658814856782556,
      "rewards/margins_min": -0.002207087352871895,
      "rewards/margins_std": 0.0027386024594306946,
      "rewards/rejected": -0.0009157261229120195,
      "step": 10
    },
    {
      "epoch": 0.01,
      "grad_norm": 0.37109375,
      "learning_rate": 7.352941176470588e-08,
      "logits/chosen": 0.007032014429569244,
      "logits/rejected": 0.7839561104774475,
      "logps/chosen": -332.548828125,
      "logps/rejected": -255.7633819580078,
      "loss": 0.6932,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.0009580751066096127,
      "rewards/margins": 0.00016147561836987734,
      "rewards/margins_max": 0.0031394611578434706,
      "rewards/margins_min": -0.0028165101539343596,
      "rewards/margins_std": 0.004211507737636566,
      "rewards/rejected": -0.0011195506667718291,
      "step": 20
    },
    {
      "epoch": 0.02,
      "grad_norm": 0.51171875,
      "learning_rate": 1.1029411764705881e-07,
      "logits/chosen": -0.04039617255330086,
      "logits/rejected": 0.607696533203125,
      "logps/chosen": -272.3553161621094,
      "logps/rejected": -256.77227783203125,
      "loss": 0.6932,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.000774255720898509,
      "rewards/margins": 8.788631384959444e-05,
      "rewards/margins_max": 0.001997451763600111,
      "rewards/margins_min": -0.0018216792959719896,
      "rewards/margins_std": 0.002700533252209425,
      "rewards/rejected": -0.0008621420711278915,
      "step": 30
    },
    {
      "epoch": 0.03,
      "grad_norm": 0.42578125,
      "learning_rate": 1.4705882352941175e-07,
      "logits/chosen": -0.06750641763210297,
      "logits/rejected": 0.6604307889938354,
      "logps/chosen": -295.84283447265625,
      "logps/rejected": -260.4482421875,
      "loss": 0.693,
      "rewards/accuracies": 0.4625000059604645,
      "rewards/chosen": -0.0007127780700102448,
      "rewards/margins": 0.00020934769418090582,
      "rewards/margins_max": 0.0021955964621156454,
      "rewards/margins_min": -0.0017769013065844774,
      "rewards/margins_std": 0.0028089801780879498,
      "rewards/rejected": -0.0009221257641911507,
      "step": 40
    },
    {
      "epoch": 0.04,
      "grad_norm": 0.470703125,
      "learning_rate": 1.8382352941176472e-07,
      "logits/chosen": -0.015060709789395332,
      "logits/rejected": 0.6939523220062256,
      "logps/chosen": -287.7695617675781,
      "logps/rejected": -237.29794311523438,
      "loss": 0.6927,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.0005769954295828938,
      "rewards/margins": 0.0007454338483512402,
      "rewards/margins_max": 0.0031318417750298977,
      "rewards/margins_min": -0.0016409741947427392,
      "rewards/margins_std": 0.0033748901914805174,
      "rewards/rejected": -0.0001684384187683463,
      "step": 50
    },
    {
      "epoch": 0.04,
      "grad_norm": 0.49609375,
      "learning_rate": 2.2058823529411763e-07,
      "logits/chosen": -0.0018195391166955233,
      "logits/rejected": 0.7142434120178223,
      "logps/chosen": -287.01068115234375,
      "logps/rejected": -256.10614013671875,
      "loss": 0.6923,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0012248576385900378,
      "rewards/margins": 0.0020126227755099535,
      "rewards/margins_max": 0.004514734260737896,
      "rewards/margins_min": -0.0004894885933026671,
      "rewards/margins_std": 0.0035385198425501585,
      "rewards/rejected": -0.0007877650787122548,
      "step": 60
    },
    {
      "epoch": 0.05,
      "grad_norm": 0.421875,
      "learning_rate": 2.5735294117647057e-07,
      "logits/chosen": 0.11708433926105499,
      "logits/rejected": 0.5532101988792419,
      "logps/chosen": -279.79791259765625,
      "logps/rejected": -243.6463623046875,
      "loss": 0.692,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": 0.0009990496328100562,
      "rewards/margins": 0.0019516434986144304,
      "rewards/margins_max": 0.004653914365917444,
      "rewards/margins_min": -0.0007506275433115661,
      "rewards/margins_std": 0.0038215883541852236,
      "rewards/rejected": -0.0009525938075967133,
      "step": 70
    },
    {
      "epoch": 0.06,
      "grad_norm": 0.50390625,
      "learning_rate": 2.941176470588235e-07,
      "logits/chosen": -0.1581064760684967,
      "logits/rejected": 0.7092851996421814,
      "logps/chosen": -311.269775390625,
      "logps/rejected": -251.24081420898438,
      "loss": 0.6915,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.002858319552615285,
      "rewards/margins": 0.0039031144697219133,
      "rewards/margins_max": 0.006772955413907766,
      "rewards/margins_min": 0.001033273758366704,
      "rewards/margins_std": 0.004058566875755787,
      "rewards/rejected": -0.001044794567860663,
      "step": 80
    },
    {
      "epoch": 0.07,
      "grad_norm": 0.390625,
      "learning_rate": 3.3088235294117644e-07,
      "logits/chosen": -0.045442212373018265,
      "logits/rejected": 0.7158526182174683,
      "logps/chosen": -265.78192138671875,
      "logps/rejected": -218.0985870361328,
      "loss": 0.691,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.003086745273321867,
      "rewards/margins": 0.0036944295279681683,
      "rewards/margins_max": 0.007027274928987026,
      "rewards/margins_min": 0.0003615838068071753,
      "rewards/margins_std": 0.0047133550979197025,
      "rewards/rejected": -0.0006076841382309794,
      "step": 90
    },
    {
      "epoch": 0.07,
      "grad_norm": 0.458984375,
      "learning_rate": 3.6764705882352943e-07,
      "logits/chosen": 0.0736229196190834,
      "logits/rejected": 0.7526043057441711,
      "logps/chosen": -253.97119140625,
      "logps/rejected": -216.62966918945312,
      "loss": 0.6907,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": 0.004654365126043558,
      "rewards/margins": 0.005358897615224123,
      "rewards/margins_max": 0.008878113701939583,
      "rewards/margins_min": 0.0018396815285086632,
      "rewards/margins_std": 0.004976923577487469,
      "rewards/rejected": -0.0007045331294648349,
      "step": 100
    },
    {
      "epoch": 0.08,
      "grad_norm": 0.42578125,
      "learning_rate": 4.044117647058823e-07,
      "logits/chosen": -0.09733302891254425,
      "logits/rejected": 0.6439273953437805,
      "logps/chosen": -289.07769775390625,
      "logps/rejected": -234.5022735595703,
      "loss": 0.6899,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 0.005760089494287968,
      "rewards/margins": 0.006674544420093298,
      "rewards/margins_max": 0.010509292595088482,
      "rewards/margins_min": 0.002839796245098114,
      "rewards/margins_std": 0.005423153750598431,
      "rewards/rejected": -0.0009144555078819394,
      "step": 110
    },
    {
      "epoch": 0.09,
      "grad_norm": 0.423828125,
      "learning_rate": 4.4117647058823526e-07,
      "logits/chosen": 0.018124500289559364,
      "logits/rejected": 0.7241690754890442,
      "logps/chosen": -311.22100830078125,
      "logps/rejected": -241.0907440185547,
      "loss": 0.6888,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": 0.008410803973674774,
      "rewards/margins": 0.007946285419166088,
      "rewards/margins_max": 0.013005959801375866,
      "rewards/margins_min": 0.002886611968278885,
      "rewards/margins_std": 0.0071554603055119514,
      "rewards/rejected": 0.000464518612716347,
      "step": 120
    },
    {
      "epoch": 0.1,
      "grad_norm": 0.494140625,
      "learning_rate": 4.779411764705882e-07,
      "logits/chosen": -0.1438770443201065,
      "logits/rejected": 0.6687305569648743,
      "logps/chosen": -265.34161376953125,
      "logps/rejected": -225.00424194335938,
      "loss": 0.6875,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": 0.011373260989785194,
      "rewards/margins": 0.01142856478691101,
      "rewards/margins_max": 0.018687166273593903,
      "rewards/margins_min": 0.004169962834566832,
      "rewards/margins_std": 0.010265212506055832,
      "rewards/rejected": -5.530398993869312e-05,
      "step": 130
    },
    {
      "epoch": 0.1,
      "grad_norm": 0.423828125,
      "learning_rate": 4.999868030671756e-07,
      "logits/chosen": 0.05402200296521187,
      "logits/rejected": 0.6528539657592773,
      "logps/chosen": -292.96112060546875,
      "logps/rejected": -259.50225830078125,
      "loss": 0.6862,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.013445700518786907,
      "rewards/margins": 0.014345765113830566,
      "rewards/margins_max": 0.022081127390265465,
      "rewards/margins_min": 0.006610400043427944,
      "rewards/margins_std": 0.010939454659819603,
      "rewards/rejected": -0.0009000620921142399,
      "step": 140
    },
    {
      "epoch": 0.11,
      "grad_norm": 0.462890625,
      "learning_rate": 4.998383535732973e-07,
      "logits/chosen": -0.16393637657165527,
      "logits/rejected": 0.45549267530441284,
      "logps/chosen": -290.1100158691406,
      "logps/rejected": -285.8172302246094,
      "loss": 0.6856,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.016459070146083832,
      "rewards/margins": 0.016152694821357727,
      "rewards/margins_max": 0.02362522855401039,
      "rewards/margins_min": 0.008680161088705063,
      "rewards/margins_std": 0.010567759163677692,
      "rewards/rejected": 0.00030637450981885195,
      "step": 150
    },
    {
      "epoch": 0.12,
      "grad_norm": 0.494140625,
      "learning_rate": 4.995250566954361e-07,
      "logits/chosen": 0.018339941278100014,
      "logits/rejected": 0.7964209914207458,
      "logps/chosen": -283.98577880859375,
      "logps/rejected": -241.65353393554688,
      "loss": 0.6842,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.01847967877984047,
      "rewards/margins": 0.01731337048113346,
      "rewards/margins_max": 0.028514724224805832,
      "rewards/margins_min": 0.006112017668783665,
      "rewards/margins_std": 0.015841105952858925,
      "rewards/rejected": 0.0011663079494610429,
      "step": 160
    },
    {
      "epoch": 0.13,
      "grad_norm": 0.5546875,
      "learning_rate": 4.990471191519357e-07,
      "logits/chosen": 0.03216102719306946,
      "logits/rejected": 0.7764664888381958,
      "logps/chosen": -279.2666931152344,
      "logps/rejected": -237.353515625,
      "loss": 0.6827,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.023826319724321365,
      "rewards/margins": 0.02013733796775341,
      "rewards/margins_max": 0.03089248575270176,
      "rewards/margins_min": 0.009382191114127636,
      "rewards/margins_std": 0.015210075303912163,
      "rewards/rejected": 0.0036889780312776566,
      "step": 170
    },
    {
      "epoch": 0.13,
      "grad_norm": 0.4453125,
      "learning_rate": 4.984048562937129e-07,
      "logits/chosen": -0.034670569002628326,
      "logits/rejected": 0.5347191095352173,
      "logps/chosen": -253.5879364013672,
      "logps/rejected": -252.5656280517578,
      "loss": 0.6827,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.021980373188853264,
      "rewards/margins": 0.020695097744464874,
      "rewards/margins_max": 0.03351113945245743,
      "rewards/margins_min": 0.007879055105149746,
      "rewards/margins_std": 0.018124621361494064,
      "rewards/rejected": 0.0012852762592956424,
      "step": 180
    },
    {
      "epoch": 0.14,
      "grad_norm": 0.37109375,
      "learning_rate": 4.975986918961825e-07,
      "logits/chosen": -0.0010436892043799162,
      "logits/rejected": 0.5325745344161987,
      "logps/chosen": -258.70465087890625,
      "logps/rejected": -254.8517608642578,
      "loss": 0.6811,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.02575596049427986,
      "rewards/margins": 0.0235120952129364,
      "rewards/margins_max": 0.037295907735824585,
      "rewards/margins_min": 0.009728283621370792,
      "rewards/margins_std": 0.019493255764245987,
      "rewards/rejected": 0.0022438622545450926,
      "step": 190
    },
    {
      "epoch": 0.15,
      "grad_norm": 0.447265625,
      "learning_rate": 4.966291578796448e-07,
      "logits/chosen": -0.08659172803163528,
      "logits/rejected": 0.8055790066719055,
      "logps/chosen": -268.54559326171875,
      "logps/rejected": -214.8706512451172,
      "loss": 0.68,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.02814793959259987,
      "rewards/margins": 0.02799884043633938,
      "rewards/margins_max": 0.042513541877269745,
      "rewards/margins_min": 0.013484140858054161,
      "rewards/margins_std": 0.020526882261037827,
      "rewards/rejected": 0.00014909838500898331,
      "step": 200
    },
    {
      "epoch": 0.15,
      "grad_norm": 0.3515625,
      "learning_rate": 4.954968939583149e-07,
      "logits/chosen": -0.15794704854488373,
      "logits/rejected": 0.6090285181999207,
      "logps/chosen": -289.7901306152344,
      "logps/rejected": -238.60183715820312,
      "loss": 0.6789,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.03186321258544922,
      "rewards/margins": 0.03159675747156143,
      "rewards/margins_max": 0.04762989282608032,
      "rewards/margins_min": 0.015563614666461945,
      "rewards/margins_std": 0.022674284875392914,
      "rewards/rejected": 0.0002664585190359503,
      "step": 210
    },
    {
      "epoch": 0.16,
      "grad_norm": 0.44921875,
      "learning_rate": 4.942026472182297e-07,
      "logits/chosen": 0.012926039285957813,
      "logits/rejected": 0.6630520820617676,
      "logps/chosen": -269.59930419921875,
      "logps/rejected": -248.7339324951172,
      "loss": 0.6778,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.0339738167822361,
      "rewards/margins": 0.03334929794073105,
      "rewards/margins_max": 0.04980397969484329,
      "rewards/margins_min": 0.016894619911909103,
      "rewards/margins_std": 0.023270433768630028,
      "rewards/rejected": 0.0006245142430998385,
      "step": 220
    },
    {
      "epoch": 0.17,
      "grad_norm": 0.43359375,
      "learning_rate": 4.92747271624308e-07,
      "logits/chosen": -0.10246638208627701,
      "logits/rejected": 0.8088499903678894,
      "logps/chosen": -293.9709167480469,
      "logps/rejected": -229.0161895751953,
      "loss": 0.6776,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.03623218461871147,
      "rewards/margins": 0.03605912998318672,
      "rewards/margins_max": 0.05394510179758072,
      "rewards/margins_min": 0.018173154443502426,
      "rewards/margins_std": 0.025294587016105652,
      "rewards/rejected": 0.0001730593794491142,
      "step": 230
    },
    {
      "epoch": 0.18,
      "grad_norm": 0.50390625,
      "learning_rate": 4.911317274568909e-07,
      "logits/chosen": -0.1043270006775856,
      "logits/rejected": 0.7763813138008118,
      "logps/chosen": -290.52032470703125,
      "logps/rejected": -238.03591918945312,
      "loss": 0.6748,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.04272182658314705,
      "rewards/margins": 0.04140879958868027,
      "rewards/margins_max": 0.0613585002720356,
      "rewards/margins_min": 0.021459100767970085,
      "rewards/margins_std": 0.0282131377607584,
      "rewards/rejected": 0.0013130264123901725,
      "step": 240
    },
    {
      "epoch": 0.18,
      "grad_norm": 0.4765625,
      "learning_rate": 4.89357080678133e-07,
      "logits/chosen": 0.042627763003110886,
      "logits/rejected": 0.5050796270370483,
      "logps/chosen": -225.12393188476562,
      "logps/rejected": -208.74288940429688,
      "loss": 0.6767,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02972862496972084,
      "rewards/margins": 0.026549100875854492,
      "rewards/margins_max": 0.04413127154111862,
      "rewards/margins_min": 0.008966929279267788,
      "rewards/margins_std": 0.02486494556069374,
      "rewards/rejected": 0.00317952549085021,
      "step": 250
    },
    {
      "epoch": 0.19,
      "grad_norm": 0.6015625,
      "learning_rate": 4.874245022286637e-07,
      "logits/chosen": 0.061648208647966385,
      "logits/rejected": 0.6458357572555542,
      "logps/chosen": -262.064208984375,
      "logps/rejected": -268.61444091796875,
      "loss": 0.677,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.03966163843870163,
      "rewards/margins": 0.036633603274822235,
      "rewards/margins_max": 0.05438825488090515,
      "rewards/margins_min": 0.01887894608080387,
      "rewards/margins_std": 0.02510887011885643,
      "rewards/rejected": 0.0030280412174761295,
      "step": 260
    },
    {
      "epoch": 0.2,
      "grad_norm": 0.3984375,
      "learning_rate": 4.853352672549815e-07,
      "logits/chosen": 0.05668522045016289,
      "logits/rejected": 0.7193032503128052,
      "logps/chosen": -261.01708984375,
      "logps/rejected": -228.1970672607422,
      "loss": 0.6732,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.038796816021203995,
      "rewards/margins": 0.038500867784023285,
      "rewards/margins_max": 0.05987783148884773,
      "rewards/margins_min": 0.01712390035390854,
      "rewards/margins_std": 0.030231595039367676,
      "rewards/rejected": 0.0002959508856292814,
      "step": 270
    },
    {
      "epoch": 0.21,
      "grad_norm": 0.3984375,
      "learning_rate": 4.830907542680918e-07,
      "logits/chosen": -0.022307882085442543,
      "logits/rejected": 0.5763358473777771,
      "logps/chosen": -259.81414794921875,
      "logps/rejected": -243.997802734375,
      "loss": 0.6746,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04487748071551323,
      "rewards/margins": 0.04191521182656288,
      "rewards/margins_max": 0.06111626699566841,
      "rewards/margins_min": 0.022714145481586456,
      "rewards/margins_std": 0.027154404670000076,
      "rewards/rejected": 0.0029622733127325773,
      "step": 280
    },
    {
      "epoch": 0.21,
      "grad_norm": 0.337890625,
      "learning_rate": 4.806924442339425e-07,
      "logits/chosen": -0.14441627264022827,
      "logits/rejected": 0.5065394639968872,
      "logps/chosen": -243.6468505859375,
      "logps/rejected": -224.4588165283203,
      "loss": 0.6735,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.045001424849033356,
      "rewards/margins": 0.04354133456945419,
      "rewards/margins_max": 0.062143802642822266,
      "rewards/margins_min": 0.02493886649608612,
      "rewards/margins_std": 0.026307862251996994,
      "rewards/rejected": 0.0014600924914702773,
      "step": 290
    },
    {
      "epoch": 0.22,
      "grad_norm": 0.4140625,
      "learning_rate": 4.781419195962598e-07,
      "logits/chosen": -0.09291721135377884,
      "logits/rejected": 0.8421966433525085,
      "logps/chosen": -294.4910583496094,
      "logps/rejected": -235.1195526123047,
      "loss": 0.6669,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05570490285754204,
      "rewards/margins": 0.05652566999197006,
      "rewards/margins_max": 0.0832170695066452,
      "rewards/margins_min": 0.02983427606523037,
      "rewards/margins_std": 0.0377473346889019,
      "rewards/rejected": -0.0008207682403735816,
      "step": 300
    },
    {
      "epoch": 0.23,
      "grad_norm": 0.427734375,
      "learning_rate": 4.754408632324253e-07,
      "logits/chosen": 0.08360351622104645,
      "logits/rejected": 0.9180965423583984,
      "logps/chosen": -274.46221923828125,
      "logps/rejected": -218.58261108398438,
      "loss": 0.6709,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.04704751446843147,
      "rewards/margins": 0.04507958143949509,
      "rewards/margins_max": 0.07067537307739258,
      "rewards/margins_min": 0.019483786076307297,
      "rewards/margins_std": 0.03619792312383652,
      "rewards/rejected": 0.0019679281394928694,
      "step": 310
    },
    {
      "epoch": 0.24,
      "grad_norm": 0.392578125,
      "learning_rate": 4.725910573430866e-07,
      "logits/chosen": -0.03652850538492203,
      "logits/rejected": 0.7102953195571899,
      "logps/chosen": -259.0872497558594,
      "logps/rejected": -220.06640625,
      "loss": 0.6666,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.052086759358644485,
      "rewards/margins": 0.05210639908909798,
      "rewards/margins_max": 0.07583759725093842,
      "rewards/margins_min": 0.02837519347667694,
      "rewards/margins_std": 0.033560991287231445,
      "rewards/rejected": -1.9636423530755565e-05,
      "step": 320
    },
    {
      "epoch": 0.24,
      "grad_norm": 0.482421875,
      "learning_rate": 4.6959438227623293e-07,
      "logits/chosen": -0.07948180288076401,
      "logits/rejected": 0.6266988515853882,
      "logps/chosen": -291.2080078125,
      "logps/rejected": -269.5301818847656,
      "loss": 0.6677,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.04915199428796768,
      "rewards/margins": 0.050243012607097626,
      "rewards/margins_max": 0.07468355447053909,
      "rewards/margins_min": 0.025802483782172203,
      "rewards/margins_std": 0.03456413000822067,
      "rewards/rejected": -0.0010910223936662078,
      "step": 330
    },
    {
      "epoch": 0.25,
      "grad_norm": 0.451171875,
      "learning_rate": 4.664528152865105e-07,
      "logits/chosen": 0.015779225155711174,
      "logits/rejected": 0.6966221332550049,
      "logps/chosen": -291.6715087890625,
      "logps/rejected": -260.85784912109375,
      "loss": 0.6643,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06338638812303543,
      "rewards/margins": 0.06281675398349762,
      "rewards/margins_max": 0.09652182459831238,
      "rewards/margins_min": 0.029111674055457115,
      "rewards/margins_std": 0.04766617342829704,
      "rewards/rejected": 0.0005696387379430234,
      "step": 340
    },
    {
      "epoch": 0.26,
      "grad_norm": 0.56640625,
      "learning_rate": 4.6316842923059816e-07,
      "logits/chosen": -0.15278920531272888,
      "logits/rejected": 0.6084717512130737,
      "logps/chosen": -247.9635009765625,
      "logps/rejected": -236.7021484375,
      "loss": 0.6618,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.07176478952169418,
      "rewards/margins": 0.07197294384241104,
      "rewards/margins_max": 0.09642891585826874,
      "rewards/margins_min": 0.047516968101263046,
      "rewards/margins_std": 0.03458596393465996,
      "rewards/rejected": -0.0002081508719129488,
      "step": 350
    },
    {
      "epoch": 0.26,
      "grad_norm": 0.45703125,
      "learning_rate": 4.5974339119950334e-07,
      "logits/chosen": -0.0013063043588772416,
      "logits/rejected": 0.6344822645187378,
      "logps/chosen": -263.31610107421875,
      "logps/rejected": -204.03660583496094,
      "loss": 0.6657,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": 0.052458666265010834,
      "rewards/margins": 0.04967236518859863,
      "rewards/margins_max": 0.07377265393733978,
      "rewards/margins_min": 0.025572070851922035,
      "rewards/margins_std": 0.034082964062690735,
      "rewards/rejected": 0.0027863027062267065,
      "step": 360
    },
    {
      "epoch": 0.27,
      "grad_norm": 0.451171875,
      "learning_rate": 4.5617996108867997e-07,
      "logits/chosen": -0.09778765588998795,
      "logits/rejected": 0.8527771234512329,
      "logps/chosen": -272.1876525878906,
      "logps/rejected": -224.5320281982422,
      "loss": 0.6615,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06909666955471039,
      "rewards/margins": 0.06635682284832001,
      "rewards/margins_max": 0.09376729279756546,
      "rewards/margins_min": 0.03894636407494545,
      "rewards/margins_std": 0.03876424580812454,
      "rewards/rejected": 0.0027398415841162205,
      "step": 370
    },
    {
      "epoch": 0.28,
      "grad_norm": 0.39453125,
      "learning_rate": 4.5248049010691304e-07,
      "logits/chosen": -0.04781482741236687,
      "logits/rejected": 0.6669384241104126,
      "logps/chosen": -270.28253173828125,
      "logps/rejected": -231.1532440185547,
      "loss": 0.6605,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06150456517934799,
      "rewards/margins": 0.0636376217007637,
      "rewards/margins_max": 0.09591880440711975,
      "rewards/margins_min": 0.03135643154382706,
      "rewards/margins_std": 0.04565249755978584,
      "rewards/rejected": -0.0021330583840608597,
      "step": 380
    },
    {
      "epoch": 0.29,
      "grad_norm": 0.5546875,
      "learning_rate": 4.486474192249533e-07,
      "logits/chosen": 0.034309376031160355,
      "logits/rejected": 0.7420383095741272,
      "logps/chosen": -246.19668579101562,
      "logps/rejected": -215.16323852539062,
      "loss": 0.6629,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.056318044662475586,
      "rewards/margins": 0.06057443469762802,
      "rewards/margins_max": 0.08743326365947723,
      "rewards/margins_min": 0.03371559828519821,
      "rewards/margins_std": 0.037984125316143036,
      "rewards/rejected": -0.0042563872411847115,
      "step": 390
    },
    {
      "epoch": 0.29,
      "grad_norm": 0.5390625,
      "learning_rate": 4.4468327756492504e-07,
      "logits/chosen": 0.09597498923540115,
      "logits/rejected": 0.7928622961044312,
      "logps/chosen": -275.2891540527344,
      "logps/rejected": -246.2664794921875,
      "loss": 0.6631,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06533468514680862,
      "rewards/margins": 0.06880095601081848,
      "rewards/margins_max": 0.1037551760673523,
      "rewards/margins_min": 0.03384672850370407,
      "rewards/margins_std": 0.04943273216485977,
      "rewards/rejected": -0.0034662713296711445,
      "step": 400
    },
    {
      "epoch": 0.3,
      "grad_norm": 0.4765625,
      "learning_rate": 4.405906807315705e-07,
      "logits/chosen": -0.021347884088754654,
      "logits/rejected": 0.6999279260635376,
      "logps/chosen": -227.08447265625,
      "logps/rejected": -222.6078643798828,
      "loss": 0.6565,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0646035447716713,
      "rewards/margins": 0.0676642507314682,
      "rewards/margins_max": 0.09568590670824051,
      "rewards/margins_min": 0.039642591029405594,
      "rewards/margins_std": 0.039628610014915466,
      "rewards/rejected": -0.003060706425458193,
      "step": 410
    },
    {
      "epoch": 0.31,
      "grad_norm": 0.41796875,
      "learning_rate": 4.363723290864314e-07,
      "logits/chosen": 0.11118383705615997,
      "logits/rejected": 0.9948932528495789,
      "logps/chosen": -271.27496337890625,
      "logps/rejected": -222.7627410888672,
      "loss": 0.6589,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07119729369878769,
      "rewards/margins": 0.07638658583164215,
      "rewards/margins_max": 0.120518758893013,
      "rewards/margins_min": 0.032254405319690704,
      "rewards/margins_std": 0.0624123141169548,
      "rewards/rejected": -0.005189282353967428,
      "step": 420
    },
    {
      "epoch": 0.32,
      "grad_norm": 0.486328125,
      "learning_rate": 4.3203100596610723e-07,
      "logits/chosen": -0.016591979190707207,
      "logits/rejected": 0.813875675201416,
      "logps/chosen": -300.334716796875,
      "logps/rejected": -259.6979064941406,
      "loss": 0.656,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.07723168283700943,
      "rewards/margins": 0.0816957950592041,
      "rewards/margins_max": 0.1194806843996048,
      "rewards/margins_min": 0.043910913169384,
      "rewards/margins_std": 0.05343589186668396,
      "rewards/rejected": -0.004464106634259224,
      "step": 430
    },
    {
      "epoch": 0.32,
      "grad_norm": 0.384765625,
      "learning_rate": 4.2756957584576436e-07,
      "logits/chosen": 0.051903001964092255,
      "logits/rejected": 0.7332680821418762,
      "logps/chosen": -291.6619873046875,
      "logps/rejected": -264.76763916015625,
      "loss": 0.6551,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07283146679401398,
      "rewards/margins": 0.07750260829925537,
      "rewards/margins_max": 0.11894078552722931,
      "rewards/margins_min": 0.03606444224715233,
      "rewards/margins_std": 0.05860241502523422,
      "rewards/rejected": -0.004671142902225256,
      "step": 440
    },
    {
      "epoch": 0.33,
      "grad_norm": 0.38671875,
      "learning_rate": 4.22990982449109e-07,
      "logits/chosen": 0.00915027130395174,
      "logits/rejected": 0.7316643595695496,
      "logps/chosen": -236.2955780029297,
      "logps/rejected": -220.3606414794922,
      "loss": 0.6569,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06984328478574753,
      "rewards/margins": 0.07647494226694107,
      "rewards/margins_max": 0.11472654342651367,
      "rewards/margins_min": 0.038223348557949066,
      "rewards/margins_std": 0.05409592390060425,
      "rewards/rejected": -0.006631660275161266,
      "step": 450
    },
    {
      "epoch": 0.34,
      "grad_norm": 0.412109375,
      "learning_rate": 4.1829824680607104e-07,
      "logits/chosen": -0.07657341659069061,
      "logits/rejected": 0.6622528433799744,
      "logps/chosen": -279.7924499511719,
      "logps/rejected": -269.6399230957031,
      "loss": 0.6549,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.07170796394348145,
      "rewards/margins": 0.0784129947423935,
      "rewards/margins_max": 0.11032243072986603,
      "rewards/margins_min": 0.046503566205501556,
      "rewards/margins_std": 0.045126743614673615,
      "rewards/rejected": -0.0067050280049443245,
      "step": 460
    },
    {
      "epoch": 0.35,
      "grad_norm": 0.453125,
      "learning_rate": 4.134944652594794e-07,
      "logits/chosen": 0.0630350112915039,
      "logits/rejected": 0.6474081873893738,
      "logps/chosen": -261.22418212890625,
      "logps/rejected": -239.87588500976562,
      "loss": 0.6581,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.05761648342013359,
      "rewards/margins": 0.06358405947685242,
      "rewards/margins_max": 0.09130324423313141,
      "rewards/margins_min": 0.03586486726999283,
      "rewards/margins_std": 0.03920084983110428,
      "rewards/rejected": -0.0059675718657672405,
      "step": 470
    },
    {
      "epoch": 0.35,
      "grad_norm": 0.392578125,
      "learning_rate": 4.085828074220451e-07,
      "logits/chosen": -0.19398275017738342,
      "logits/rejected": 0.5935221910476685,
      "logps/chosen": -270.58892822265625,
      "logps/rejected": -245.28076171875,
      "loss": 0.6561,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06964448094367981,
      "rewards/margins": 0.08161848783493042,
      "rewards/margins_max": 0.12005958706140518,
      "rewards/margins_min": 0.04317738488316536,
      "rewards/margins_std": 0.0543639250099659,
      "rewards/rejected": -0.011974002234637737,
      "step": 480
    },
    {
      "epoch": 0.36,
      "grad_norm": 0.373046875,
      "learning_rate": 4.035665140849994e-07,
      "logits/chosen": -0.14574551582336426,
      "logits/rejected": 0.5638002157211304,
      "logps/chosen": -274.3948669433594,
      "logps/rejected": -228.3678436279297,
      "loss": 0.6602,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06451380997896194,
      "rewards/margins": 0.07035050541162491,
      "rewards/margins_max": 0.10534896701574326,
      "rewards/margins_min": 0.035352036356925964,
      "rewards/margins_std": 0.04949530214071274,
      "rewards/rejected": -0.005836695898324251,
      "step": 490
    },
    {
      "epoch": 0.37,
      "grad_norm": 0.470703125,
      "learning_rate": 3.984488950797678e-07,
      "logits/chosen": -0.040041010826826096,
      "logits/rejected": 0.7338499426841736,
      "logps/chosen": -265.7203063964844,
      "logps/rejected": -230.03170776367188,
      "loss": 0.6557,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07825618237257004,
      "rewards/margins": 0.07796762883663177,
      "rewards/margins_max": 0.11045694351196289,
      "rewards/margins_min": 0.04547831788659096,
      "rewards/margins_std": 0.04594682529568672,
      "rewards/rejected": 0.00028855842538177967,
      "step": 500
    },
    {
      "epoch": 0.38,
      "grad_norm": 0.46484375,
      "learning_rate": 3.9323332709408904e-07,
      "logits/chosen": 0.02443014457821846,
      "logits/rejected": 0.604122519493103,
      "logps/chosen": -254.78701782226562,
      "logps/rejected": -233.02084350585938,
      "loss": 0.6547,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.060421328991651535,
      "rewards/margins": 0.07197172939777374,
      "rewards/margins_max": 0.11490993201732635,
      "rewards/margins_min": 0.029033523052930832,
      "rewards/margins_std": 0.060723792761564255,
      "rewards/rejected": -0.01155039481818676,
      "step": 510
    },
    {
      "epoch": 0.38,
      "grad_norm": 0.40234375,
      "learning_rate": 3.879232514440227e-07,
      "logits/chosen": 0.04711779206991196,
      "logits/rejected": 0.9458179473876953,
      "logps/chosen": -280.1850280761719,
      "logps/rejected": -221.70785522460938,
      "loss": 0.6533,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.07137273252010345,
      "rewards/margins": 0.0817800909280777,
      "rewards/margins_max": 0.12524864077568054,
      "rewards/margins_min": 0.038311541080474854,
      "rewards/margins_std": 0.06147381663322449,
      "rewards/rejected": -0.01040735375136137,
      "step": 520
    },
    {
      "epoch": 0.39,
      "grad_norm": 0.51171875,
      "learning_rate": 3.825221718033129e-07,
      "logits/chosen": 0.09109713137149811,
      "logits/rejected": 0.7228142023086548,
      "logps/chosen": -256.89202880859375,
      "logps/rejected": -236.75460815429688,
      "loss": 0.6552,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06750547885894775,
      "rewards/margins": 0.0867571011185646,
      "rewards/margins_max": 0.12318144738674164,
      "rewards/margins_min": 0.050332747399806976,
      "rewards/margins_std": 0.05151181295514107,
      "rewards/rejected": -0.019251624122262,
      "step": 530
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.375,
      "learning_rate": 3.7703365189160746e-07,
      "logits/chosen": -0.09472191333770752,
      "logits/rejected": 0.6297842264175415,
      "logps/chosen": -261.3214416503906,
      "logps/rejected": -244.3570556640625,
      "loss": 0.6562,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.07096521556377411,
      "rewards/margins": 0.07419043034315109,
      "rewards/margins_max": 0.1066773533821106,
      "rewards/margins_min": 0.041703514754772186,
      "rewards/margins_std": 0.04594343155622482,
      "rewards/rejected": -0.0032252087257802486,
      "step": 540
    },
    {
      "epoch": 0.4,
      "grad_norm": 0.396484375,
      "learning_rate": 3.714613131230587e-07,
      "logits/chosen": -0.002119688782840967,
      "logits/rejected": 0.6269344091415405,
      "logps/chosen": -249.4778594970703,
      "logps/rejected": -223.02603149414062,
      "loss": 0.6572,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.06649978458881378,
      "rewards/margins": 0.07572688907384872,
      "rewards/margins_max": 0.10936542600393295,
      "rewards/margins_min": 0.0420883484184742,
      "rewards/margins_std": 0.04757208377122879,
      "rewards/rejected": -0.009227104485034943,
      "step": 550
    },
    {
      "epoch": 0.41,
      "grad_norm": 0.427734375,
      "learning_rate": 3.6580883221685533e-07,
      "logits/chosen": 0.05928317829966545,
      "logits/rejected": 0.6542156338691711,
      "logps/chosen": -249.701171875,
      "logps/rejected": -235.28256225585938,
      "loss": 0.6546,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06236664578318596,
      "rewards/margins": 0.07323845475912094,
      "rewards/margins_max": 0.11050041764974594,
      "rewards/margins_min": 0.03597649186849594,
      "rewards/margins_std": 0.05269637703895569,
      "rewards/rejected": -0.010871809907257557,
      "step": 560
    },
    {
      "epoch": 0.42,
      "grad_norm": 0.39453125,
      "learning_rate": 3.6007993877126386e-07,
      "logits/chosen": -0.15580764412879944,
      "logits/rejected": 0.7158504724502563,
      "logps/chosen": -291.7669372558594,
      "logps/rejected": -232.95767211914062,
      "loss": 0.6561,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": 0.06417657434940338,
      "rewards/margins": 0.07455410063266754,
      "rewards/margins_max": 0.11633304506540298,
      "rewards/margins_min": 0.032775141298770905,
      "rewards/margins_std": 0.05908435583114624,
      "rewards/rejected": -0.010377511382102966,
      "step": 570
    },
    {
      "epoch": 0.43,
      "grad_norm": 0.490234375,
      "learning_rate": 3.5427841280277937e-07,
      "logits/chosen": 0.07802899181842804,
      "logits/rejected": 0.6800590753555298,
      "logps/chosen": -253.85107421875,
      "logps/rejected": -234.7869110107422,
      "loss": 0.6586,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": 0.058535121381282806,
      "rewards/margins": 0.06843312084674835,
      "rewards/margins_max": 0.10552020370960236,
      "rewards/margins_min": 0.03134603053331375,
      "rewards/margins_std": 0.0524490587413311,
      "rewards/rejected": -0.0098979901522398,
      "step": 580
    },
    {
      "epoch": 0.43,
      "grad_norm": 0.482421875,
      "learning_rate": 3.484080822520096e-07,
      "logits/chosen": -0.049510225653648376,
      "logits/rejected": 0.7604032754898071,
      "logps/chosen": -268.8267822265625,
      "logps/rejected": -224.325439453125,
      "loss": 0.654,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07341635227203369,
      "rewards/margins": 0.07777970284223557,
      "rewards/margins_max": 0.11911604553461075,
      "rewards/margins_min": 0.03644336014986038,
      "rewards/margins_std": 0.058458417654037476,
      "rewards/rejected": -0.004363355692476034,
      "step": 590
    },
    {
      "epoch": 0.44,
      "grad_norm": 0.400390625,
      "learning_rate": 3.4247282045793797e-07,
      "logits/chosen": -0.19325688481330872,
      "logits/rejected": 0.6270472407341003,
      "logps/chosen": -282.7298889160156,
      "logps/rejected": -252.6767578125,
      "loss": 0.6482,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.07601656764745712,
      "rewards/margins": 0.09246112406253815,
      "rewards/margins_max": 0.13728852570056915,
      "rewards/margins_min": 0.04763372614979744,
      "rewards/margins_std": 0.06339551508426666,
      "rewards/rejected": -0.016444550827145576,
      "step": 600
    },
    {
      "epoch": 0.45,
      "grad_norm": 0.458984375,
      "learning_rate": 3.3647654360223144e-07,
      "logits/chosen": -0.05807774141430855,
      "logits/rejected": 0.6809174418449402,
      "logps/chosen": -299.19000244140625,
      "logps/rejected": -262.0146179199219,
      "loss": 0.6554,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06890231370925903,
      "rewards/margins": 0.07707327604293823,
      "rewards/margins_max": 0.11320241540670395,
      "rewards/margins_min": 0.04094412922859192,
      "rewards/margins_std": 0.05109432339668274,
      "rewards/rejected": -0.0081709586083889,
      "step": 610
    },
    {
      "epoch": 0.46,
      "grad_norm": 0.4609375,
      "learning_rate": 3.30423208125281e-07,
      "logits/chosen": -0.15423426032066345,
      "logits/rejected": 0.6746962070465088,
      "logps/chosen": -361.3031921386719,
      "logps/rejected": -271.8617858886719,
      "loss": 0.6527,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.08182433992624283,
      "rewards/margins": 0.08147110044956207,
      "rewards/margins_max": 0.1197957769036293,
      "rewards/margins_min": 0.04314642399549484,
      "rewards/margins_std": 0.05419927090406418,
      "rewards/rejected": 0.00035324436612427235,
      "step": 620
    },
    {
      "epoch": 0.46,
      "grad_norm": 0.53515625,
      "learning_rate": 3.2431680811567833e-07,
      "logits/chosen": -0.18638429045677185,
      "logits/rejected": 0.5596809387207031,
      "logps/chosen": -265.38189697265625,
      "logps/rejected": -216.4676513671875,
      "loss": 0.6498,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07413068413734436,
      "rewards/margins": 0.08364250510931015,
      "rewards/margins_max": 0.11661893129348755,
      "rewards/margins_min": 0.05066608637571335,
      "rewards/margins_std": 0.046635713428258896,
      "rewards/rejected": -0.009511834010481834,
      "step": 630
    },
    {
      "epoch": 0.47,
      "grad_norm": 0.4765625,
      "learning_rate": 3.1816137267485136e-07,
      "logits/chosen": -0.01266663521528244,
      "logits/rejected": 0.7700913548469543,
      "logps/chosen": -270.8904113769531,
      "logps/rejected": -227.817138671875,
      "loss": 0.6499,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.08034569025039673,
      "rewards/margins": 0.09530878067016602,
      "rewards/margins_max": 0.13598743081092834,
      "rewards/margins_min": 0.05463013797998428,
      "rewards/margins_std": 0.0575282946228981,
      "rewards/rejected": -0.014963088557124138,
      "step": 640
    },
    {
      "epoch": 0.48,
      "grad_norm": 0.458984375,
      "learning_rate": 3.1196096325859815e-07,
      "logits/chosen": -0.09016970545053482,
      "logits/rejected": 0.5758002400398254,
      "logps/chosen": -248.17721557617188,
      "logps/rejected": -231.0363311767578,
      "loss": 0.6539,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.06288131326436996,
      "rewards/margins": 0.08216816931962967,
      "rewards/margins_max": 0.12130717188119888,
      "rewards/margins_min": 0.04302916303277016,
      "rewards/margins_std": 0.05535091087222099,
      "rewards/rejected": -0.019286850467324257,
      "step": 650
    },
    {
      "epoch": 0.49,
      "grad_norm": 0.357421875,
      "learning_rate": 3.057196709972727e-07,
      "logits/chosen": -0.06700615584850311,
      "logits/rejected": 0.7238128781318665,
      "logps/chosen": -273.21044921875,
      "logps/rejected": -222.5702667236328,
      "loss": 0.6541,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.07693549245595932,
      "rewards/margins": 0.08857451379299164,
      "rewards/margins_max": 0.13141193985939026,
      "rewards/margins_min": 0.045737091451883316,
      "rewards/margins_std": 0.0605812594294548,
      "rewards/rejected": -0.011639016680419445,
      "step": 660
    },
    {
      "epoch": 0.49,
      "grad_norm": 0.384765625,
      "learning_rate": 2.9944161399639086e-07,
      "logits/chosen": -0.17069509625434875,
      "logits/rejected": 0.5526743531227112,
      "logps/chosen": -274.2879943847656,
      "logps/rejected": -238.9512176513672,
      "loss": 0.6554,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.07316674292087555,
      "rewards/margins": 0.0783548504114151,
      "rewards/margins_max": 0.12053277343511581,
      "rewards/margins_min": 0.03617694973945618,
      "rewards/margins_std": 0.059648580849170685,
      "rewards/rejected": -0.0051881177350878716,
      "step": 670
    },
    {
      "epoch": 0.5,
      "grad_norm": 0.4453125,
      "learning_rate": 2.9313093461943824e-07,
      "logits/chosen": -0.16692019999027252,
      "logits/rejected": 0.6132463216781616,
      "logps/chosen": -266.98297119140625,
      "logps/rejected": -240.8970489501953,
      "loss": 0.6514,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.0738702192902565,
      "rewards/margins": 0.08596795052289963,
      "rewards/margins_max": 0.12484346330165863,
      "rewards/margins_min": 0.04709243029356003,
      "rewards/margins_std": 0.05497828125953674,
      "rewards/rejected": -0.012097733095288277,
      "step": 680
    },
    {
      "epoch": 0.51,
      "grad_norm": 0.404296875,
      "learning_rate": 2.8679179675467104e-07,
      "logits/chosen": -0.028652098029851913,
      "logits/rejected": 0.7112724781036377,
      "logps/chosen": -263.5270080566406,
      "logps/rejected": -232.5438232421875,
      "loss": 0.6505,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07920118421316147,
      "rewards/margins": 0.08954565227031708,
      "rewards/margins_max": 0.13445377349853516,
      "rewards/margins_min": 0.04463755339384079,
      "rewards/margins_std": 0.06350965052843094,
      "rewards/rejected": -0.010344477370381355,
      "step": 690
    },
    {
      "epoch": 0.52,
      "grad_norm": 0.44140625,
      "learning_rate": 2.80428383067716e-07,
      "logits/chosen": 0.04036666825413704,
      "logits/rejected": 0.5802733302116394,
      "logps/chosen": -276.1773681640625,
      "logps/rejected": -284.2413024902344,
      "loss": 0.6492,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07053406536579132,
      "rewards/margins": 0.09340489655733109,
      "rewards/margins_max": 0.13880935311317444,
      "rewards/margins_min": 0.048000432550907135,
      "rewards/margins_std": 0.06421160697937012,
      "rewards/rejected": -0.022870829328894615,
      "step": 700
    },
    {
      "epoch": 0.52,
      "grad_norm": 0.484375,
      "learning_rate": 2.7404489224177973e-07,
      "logits/chosen": -0.10626095533370972,
      "logits/rejected": 0.6656783819198608,
      "logps/chosen": -273.10369873046875,
      "logps/rejected": -240.2863006591797,
      "loss": 0.6555,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.06227077171206474,
      "rewards/margins": 0.07790277898311615,
      "rewards/margins_max": 0.12013592571020126,
      "rewards/margins_min": 0.03566960617899895,
      "rewards/margins_std": 0.05972672253847122,
      "rewards/rejected": -0.015631992369890213,
      "step": 710
    },
    {
      "epoch": 0.53,
      "grad_norm": 0.392578125,
      "learning_rate": 2.676455362072894e-07,
      "logits/chosen": -0.09553918987512589,
      "logits/rejected": 0.7383521795272827,
      "logps/chosen": -302.948974609375,
      "logps/rejected": -269.911376953125,
      "loss": 0.6506,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.08486850559711456,
      "rewards/margins": 0.10140830278396606,
      "rewards/margins_max": 0.14273163676261902,
      "rewards/margins_min": 0.06008495017886162,
      "rewards/margins_std": 0.05844002962112427,
      "rewards/rejected": -0.016539786010980606,
      "step": 720
    },
    {
      "epoch": 0.54,
      "grad_norm": 0.37890625,
      "learning_rate": 2.612345373627937e-07,
      "logits/chosen": 0.11825668811798096,
      "logits/rejected": 0.7459693551063538,
      "logps/chosen": -275.9749450683594,
      "logps/rejected": -238.9975128173828,
      "loss": 0.6551,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06423775851726532,
      "rewards/margins": 0.08155209571123123,
      "rewards/margins_max": 0.12171584367752075,
      "rewards/margins_min": 0.04138834401965141,
      "rewards/margins_std": 0.05680011585354805,
      "rewards/rejected": -0.017314324155449867,
      "step": 730
    },
    {
      "epoch": 0.54,
      "grad_norm": 0.40234375,
      "learning_rate": 2.54816125788955e-07,
      "logits/chosen": -0.04539821669459343,
      "logits/rejected": 0.8194332122802734,
      "logps/chosen": -290.15557861328125,
      "logps/rejected": -230.5435333251953,
      "loss": 0.6515,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.074457548558712,
      "rewards/margins": 0.08972884714603424,
      "rewards/margins_max": 0.13167297840118408,
      "rewards/margins_min": 0.04778473079204559,
      "rewards/margins_std": 0.059317946434020996,
      "rewards/rejected": -0.015271296724677086,
      "step": 740
    },
    {
      "epoch": 0.55,
      "grad_norm": 0.38671875,
      "learning_rate": 2.4839453645747467e-07,
      "logits/chosen": 0.04359906539320946,
      "logits/rejected": 0.622580349445343,
      "logps/chosen": -276.761474609375,
      "logps/rejected": -252.15261840820312,
      "loss": 0.6543,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.07819774001836777,
      "rewards/margins": 0.0873616486787796,
      "rewards/margins_max": 0.14198613166809082,
      "rewards/margins_min": 0.03273715078830719,
      "rewards/margins_std": 0.07725068926811218,
      "rewards/rejected": -0.009163893759250641,
      "step": 750
    },
    {
      "epoch": 0.56,
      "grad_norm": 0.33984375,
      "learning_rate": 2.4197400643678987e-07,
      "logits/chosen": 0.03581676632165909,
      "logits/rejected": 0.8077651858329773,
      "logps/chosen": -280.2861328125,
      "logps/rejected": -233.9840545654297,
      "loss": 0.6528,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.07635350525379181,
      "rewards/margins": 0.09026426076889038,
      "rewards/margins_max": 0.1272718459367752,
      "rewards/margins_min": 0.05325668305158615,
      "rewards/margins_std": 0.05233662202954292,
      "rewards/rejected": -0.013910764828324318,
      "step": 760
    },
    {
      "epoch": 0.57,
      "grad_norm": 0.435546875,
      "learning_rate": 2.3555877209638726e-07,
      "logits/chosen": -0.12947851419448853,
      "logits/rejected": 0.7489079833030701,
      "logps/chosen": -290.2663879394531,
      "logps/rejected": -254.322021484375,
      "loss": 0.6506,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0825168713927269,
      "rewards/margins": 0.08864180743694305,
      "rewards/margins_max": 0.12674041092395782,
      "rewards/margins_min": 0.050543200224637985,
      "rewards/margins_std": 0.05387955904006958,
      "rewards/rejected": -0.006124935112893581,
      "step": 770
    },
    {
      "epoch": 0.57,
      "grad_norm": 0.390625,
      "learning_rate": 2.2915306631157817e-07,
      "logits/chosen": -0.04638870805501938,
      "logits/rejected": 0.6960999965667725,
      "logps/chosen": -238.04745483398438,
      "logps/rejected": -215.9326629638672,
      "loss": 0.6497,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06742905080318451,
      "rewards/margins": 0.08723022788763046,
      "rewards/margins_max": 0.13323740661144257,
      "rewards/margins_min": 0.04122306779026985,
      "rewards/margins_std": 0.0650639608502388,
      "rewards/rejected": -0.019801190122961998,
      "step": 780
    },
    {
      "epoch": 0.58,
      "grad_norm": 0.451171875,
      "learning_rate": 2.2276111567057887e-07,
      "logits/chosen": 0.19736722111701965,
      "logits/rejected": 0.8112510442733765,
      "logps/chosen": -263.0642395019531,
      "logps/rejected": -247.5417022705078,
      "loss": 0.6481,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07030580937862396,
      "rewards/margins": 0.08302759379148483,
      "rewards/margins_max": 0.12399895489215851,
      "rewards/margins_min": 0.042056240141391754,
      "rewards/margins_std": 0.05794224888086319,
      "rewards/rejected": -0.01272178627550602,
      "step": 790
    },
    {
      "epoch": 0.59,
      "grad_norm": 0.474609375,
      "learning_rate": 2.1638713768573936e-07,
      "logits/chosen": 0.0460067018866539,
      "logits/rejected": 0.6669069528579712,
      "logps/chosen": -227.3536834716797,
      "logps/rejected": -220.83984375,
      "loss": 0.6476,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.0669163167476654,
      "rewards/margins": 0.07873805612325668,
      "rewards/margins_max": 0.12541484832763672,
      "rewards/margins_min": 0.03206125646829605,
      "rewards/margins_std": 0.0660109594464302,
      "rewards/rejected": -0.011821739375591278,
      "step": 800
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.484375,
      "learning_rate": 2.100353380107609e-07,
      "logits/chosen": -0.007617829833179712,
      "logits/rejected": 0.7070167660713196,
      "logps/chosen": -271.02691650390625,
      "logps/rejected": -219.64944458007812,
      "loss": 0.655,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.05975126102566719,
      "rewards/margins": 0.07805275171995163,
      "rewards/margins_max": 0.11311063915491104,
      "rewards/margins_min": 0.04299486428499222,
      "rewards/margins_std": 0.04957933351397514,
      "rewards/rejected": -0.01830149069428444,
      "step": 810
    },
    {
      "epoch": 0.6,
      "grad_norm": 0.400390625,
      "learning_rate": 2.0370990766573698e-07,
      "logits/chosen": 0.11678284406661987,
      "logits/rejected": 0.8694367408752441,
      "logps/chosen": -319.4522705078125,
      "logps/rejected": -231.9202423095703,
      "loss": 0.653,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.060270898044109344,
      "rewards/margins": 0.08039253205060959,
      "rewards/margins_max": 0.11875177919864655,
      "rewards/margins_min": 0.042033277451992035,
      "rewards/margins_std": 0.054248176515102386,
      "rewards/rejected": -0.020121637731790543,
      "step": 820
    },
    {
      "epoch": 0.61,
      "grad_norm": 0.357421875,
      "learning_rate": 1.974150202718513e-07,
      "logits/chosen": 0.037704307585954666,
      "logits/rejected": 0.6765593886375427,
      "logps/chosen": -252.72128295898438,
      "logps/rejected": -251.72232055664062,
      "loss": 0.6551,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06962188333272934,
      "rewards/margins": 0.07926660031080246,
      "rewards/margins_max": 0.11337349563837051,
      "rewards/margins_min": 0.0451597236096859,
      "rewards/margins_std": 0.04823441430926323,
      "rewards/rejected": -0.009644721634685993,
      "step": 830
    },
    {
      "epoch": 0.62,
      "grad_norm": 0.4296875,
      "learning_rate": 1.9115482929755445e-07,
      "logits/chosen": 0.21195778250694275,
      "logits/rejected": 0.8628422021865845,
      "logps/chosen": -255.17141723632812,
      "logps/rejected": -230.54837036132812,
      "loss": 0.6542,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.05505312606692314,
      "rewards/margins": 0.08028857409954071,
      "rewards/margins_max": 0.12079598009586334,
      "rewards/margins_min": 0.03978116437792778,
      "rewards/margins_std": 0.05728612467646599,
      "rewards/rejected": -0.025235449895262718,
      "step": 840
    },
    {
      "epoch": 0.63,
      "grad_norm": 0.451171875,
      "learning_rate": 1.8493346531803887e-07,
      "logits/chosen": -0.0924936980009079,
      "logits/rejected": 0.6660225987434387,
      "logps/chosen": -253.78762817382812,
      "logps/rejected": -230.6340789794922,
      "loss": 0.654,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.07504816353321075,
      "rewards/margins": 0.08388366550207138,
      "rewards/margins_max": 0.13147205114364624,
      "rewards/margins_min": 0.03629531338810921,
      "rewards/margins_std": 0.06730011850595474,
      "rewards/rejected": -0.00883551873266697,
      "step": 850
    },
    {
      "epoch": 0.63,
      "grad_norm": 0.62890625,
      "learning_rate": 1.7875503328981807e-07,
      "logits/chosen": 0.03313580900430679,
      "logits/rejected": 0.7846869826316833,
      "logps/chosen": -289.10919189453125,
      "logps/rejected": -260.46209716796875,
      "loss": 0.651,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.07691006362438202,
      "rewards/margins": 0.09373887628316879,
      "rewards/margins_max": 0.14286434650421143,
      "rewards/margins_min": 0.04461339861154556,
      "rewards/margins_std": 0.06947391480207443,
      "rewards/rejected": -0.01682879962027073,
      "step": 860
    },
    {
      "epoch": 0.64,
      "grad_norm": 0.4765625,
      "learning_rate": 1.7262360984221006e-07,
      "logits/chosen": 0.07170330733060837,
      "logits/rejected": 0.585077166557312,
      "logps/chosen": -232.71224975585938,
      "logps/rejected": -241.5399932861328,
      "loss": 0.649,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06494490802288055,
      "rewards/margins": 0.082070492208004,
      "rewards/margins_max": 0.11704879999160767,
      "rewards/margins_min": 0.04709218442440033,
      "rewards/margins_std": 0.049466803669929504,
      "rewards/rejected": -0.017125584185123444,
      "step": 870
    },
    {
      "epoch": 0.65,
      "grad_norm": 0.46875,
      "learning_rate": 1.6654324058751175e-07,
      "logits/chosen": -0.03041740693151951,
      "logits/rejected": 0.655995786190033,
      "logps/chosen": -307.1095275878906,
      "logps/rejected": -223.6023712158203,
      "loss": 0.6511,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.05577283352613449,
      "rewards/margins": 0.0676901713013649,
      "rewards/margins_max": 0.10265400260686874,
      "rewards/margins_min": 0.03272632881999016,
      "rewards/margins_std": 0.04944632947444916,
      "rewards/rejected": -0.011917328462004662,
      "step": 880
    },
    {
      "epoch": 0.65,
      "grad_norm": 0.439453125,
      "learning_rate": 1.6051793745163812e-07,
      "logits/chosen": -0.03592555597424507,
      "logits/rejected": 0.5853763222694397,
      "logps/chosen": -272.7641296386719,
      "logps/rejected": -243.679443359375,
      "loss": 0.6543,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07209287583827972,
      "rewards/margins": 0.0827474445104599,
      "rewards/margins_max": 0.11978437751531601,
      "rewards/margins_min": 0.045710500329732895,
      "rewards/margins_std": 0.05237815901637077,
      "rewards/rejected": -0.010654577054083347,
      "step": 890
    },
    {
      "epoch": 0.66,
      "grad_norm": 0.435546875,
      "learning_rate": 1.5455167602698915e-07,
      "logits/chosen": 0.10257561504840851,
      "logits/rejected": 0.7078714370727539,
      "logps/chosen": -239.56201171875,
      "logps/rejected": -240.7050323486328,
      "loss": 0.6508,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06524372100830078,
      "rewards/margins": 0.08618183434009552,
      "rewards/margins_max": 0.12894438207149506,
      "rewards/margins_min": 0.043419282883405685,
      "rewards/margins_std": 0.060475386679172516,
      "rewards/rejected": -0.020938115194439888,
      "step": 900
    },
    {
      "epoch": 0.67,
      "grad_norm": 0.5,
      "learning_rate": 1.4864839294928924e-07,
      "logits/chosen": 0.03240091726183891,
      "logits/rejected": 0.7821232080459595,
      "logps/chosen": -283.28338623046875,
      "logps/rejected": -228.65771484375,
      "loss": 0.6537,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06496749818325043,
      "rewards/margins": 0.08208582550287247,
      "rewards/margins_max": 0.12210579216480255,
      "rewards/margins_min": 0.04206586629152298,
      "rewards/margins_std": 0.056596774607896805,
      "rewards/rejected": -0.01711832545697689,
      "step": 910
    },
    {
      "epoch": 0.68,
      "grad_norm": 0.515625,
      "learning_rate": 1.428119833001315e-07,
      "logits/chosen": -0.23715606331825256,
      "logits/rejected": 0.6829497218132019,
      "logps/chosen": -352.04071044921875,
      "logps/rejected": -288.7782287597656,
      "loss": 0.6512,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.07856696844100952,
      "rewards/margins": 0.08786121755838394,
      "rewards/margins_max": 0.1261076182126999,
      "rewards/margins_min": 0.049614839255809784,
      "rewards/margins_std": 0.05408855527639389,
      "rewards/rejected": -0.009294259361922741,
      "step": 920
    },
    {
      "epoch": 0.68,
      "grad_norm": 0.41796875,
      "learning_rate": 1.370462980369401e-07,
      "logits/chosen": 0.029560232535004616,
      "logits/rejected": 0.729602038860321,
      "logps/chosen": -257.4678955078125,
      "logps/rejected": -246.0548858642578,
      "loss": 0.6487,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.0831000953912735,
      "rewards/margins": 0.09744415432214737,
      "rewards/margins_max": 0.15836606919765472,
      "rewards/margins_min": 0.03652223199605942,
      "rewards/margins_std": 0.08615661412477493,
      "rewards/rejected": -0.014344056136906147,
      "step": 930
    },
    {
      "epoch": 0.69,
      "grad_norm": 0.4453125,
      "learning_rate": 1.3135514145204606e-07,
      "logits/chosen": 0.046013571321964264,
      "logits/rejected": 0.5954084396362305,
      "logps/chosen": -269.13629150390625,
      "logps/rejected": -249.99008178710938,
      "loss": 0.6511,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06792394816875458,
      "rewards/margins": 0.08472023159265518,
      "rewards/margins_max": 0.12758344411849976,
      "rewards/margins_min": 0.0418570414185524,
      "rewards/margins_std": 0.06061771512031555,
      "rewards/rejected": -0.016796285286545753,
      "step": 940
    },
    {
      "epoch": 0.7,
      "grad_norm": 0.515625,
      "learning_rate": 1.257422686625539e-07,
      "logits/chosen": -0.12281881272792816,
      "logits/rejected": 0.7430060505867004,
      "logps/chosen": -328.2705383300781,
      "logps/rejected": -239.69070434570312,
      "loss": 0.6552,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06796546280384064,
      "rewards/margins": 0.08045408874750137,
      "rewards/margins_max": 0.1251390427350998,
      "rewards/margins_min": 0.03576914221048355,
      "rewards/margins_std": 0.06319407373666763,
      "rewards/rejected": -0.01248864270746708,
      "step": 950
    },
    {
      "epoch": 0.71,
      "grad_norm": 0.49609375,
      "learning_rate": 1.2021138313265444e-07,
      "logits/chosen": -0.05693725496530533,
      "logits/rejected": 0.5191583633422852,
      "logps/chosen": -267.0467224121094,
      "logps/rejected": -253.3563995361328,
      "loss": 0.6547,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06574182212352753,
      "rewards/margins": 0.0813496932387352,
      "rewards/margins_max": 0.11537833511829376,
      "rewards/margins_min": 0.04732104018330574,
      "rewards/margins_std": 0.048123784363269806,
      "rewards/rejected": -0.015607871115207672,
      "step": 960
    },
    {
      "epoch": 0.71,
      "grad_norm": 0.421875,
      "learning_rate": 1.1476613423001974e-07,
      "logits/chosen": 0.01074630580842495,
      "logits/rejected": 0.7058780789375305,
      "logps/chosen": -268.10162353515625,
      "logps/rejected": -246.88095092773438,
      "loss": 0.6503,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.06480704247951508,
      "rewards/margins": 0.07971958816051483,
      "rewards/margins_max": 0.12575621902942657,
      "rewards/margins_min": 0.03368296101689339,
      "rewards/margins_std": 0.0651056244969368,
      "rewards/rejected": -0.014912550337612629,
      "step": 970
    },
    {
      "epoch": 0.72,
      "grad_norm": 0.447265625,
      "learning_rate": 1.0941011481789042e-07,
      "logits/chosen": 0.16233515739440918,
      "logits/rejected": 0.6619356274604797,
      "logps/chosen": -268.57537841796875,
      "logps/rejected": -248.046142578125,
      "loss": 0.6547,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.061273593455553055,
      "rewards/margins": 0.07566945254802704,
      "rewards/margins_max": 0.1124287024140358,
      "rewards/margins_min": 0.03891021013259888,
      "rewards/margins_std": 0.051985424011945724,
      "rewards/rejected": -0.014395867474377155,
      "step": 980
    },
    {
      "epoch": 0.73,
      "grad_norm": 0.4296875,
      "learning_rate": 1.041468588844476e-07,
      "logits/chosen": 0.021765699610114098,
      "logits/rejected": 0.5723878145217896,
      "logps/chosen": -252.99365234375,
      "logps/rejected": -237.05233764648438,
      "loss": 0.6506,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.06569187343120575,
      "rewards/margins": 0.08357974141836166,
      "rewards/margins_max": 0.12318392097949982,
      "rewards/margins_min": 0.043975573033094406,
      "rewards/margins_std": 0.056008756160736084,
      "rewards/rejected": -0.017887871712446213,
      "step": 990
    },
    {
      "epoch": 0.74,
      "grad_norm": 0.474609375,
      "learning_rate": 9.897983921102954e-08,
      "logits/chosen": -0.08432263880968094,
      "logits/rejected": 0.5865111351013184,
      "logps/chosen": -265.72735595703125,
      "logps/rejected": -234.95028686523438,
      "loss": 0.6572,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.060374390333890915,
      "rewards/margins": 0.070238396525383,
      "rewards/margins_max": 0.10305947065353394,
      "rewards/margins_min": 0.037417326122522354,
      "rewards/margins_std": 0.04641600325703621,
      "rewards/rejected": -0.009864002466201782,
      "step": 1000
    },
    {
      "epoch": 0.74,
      "grad_norm": 0.42578125,
      "learning_rate": 9.391246508073433e-08,
      "logits/chosen": -0.17238160967826843,
      "logits/rejected": 0.6868064999580383,
      "logps/chosen": -321.6485595703125,
      "logps/rejected": -263.62188720703125,
      "loss": 0.6505,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07812955230474472,
      "rewards/margins": 0.09409363567829132,
      "rewards/margins_max": 0.13336913287639618,
      "rewards/margins_min": 0.054818131029605865,
      "rewards/margins_std": 0.05554394796490669,
      "rewards/rejected": -0.015964074060320854,
      "step": 1010
    },
    {
      "epoch": 0.75,
      "grad_norm": 0.408203125,
      "learning_rate": 8.894808002892037e-08,
      "logits/chosen": -0.048636335879564285,
      "logits/rejected": 0.6718664765357971,
      "logps/chosen": -286.724365234375,
      "logps/rejected": -256.17767333984375,
      "loss": 0.6521,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.07918758690357208,
      "rewards/margins": 0.09503775835037231,
      "rewards/margins_max": 0.1390329748392105,
      "rewards/margins_min": 0.05104254558682442,
      "rewards/margins_std": 0.06221862882375717,
      "rewards/rejected": -0.015850163996219635,
      "step": 1020
    },
    {
      "epoch": 0.76,
      "grad_norm": 0.423828125,
      "learning_rate": 8.408995963708756e-08,
      "logits/chosen": -0.03906983509659767,
      "logits/rejected": 0.7512571215629578,
      "logps/chosen": -289.33660888671875,
      "logps/rejected": -260.1453857421875,
      "loss": 0.6546,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.07737907022237778,
      "rewards/margins": 0.08742411434650421,
      "rewards/margins_max": 0.12180934101343155,
      "rewards/margins_min": 0.053038887679576874,
      "rewards/margins_std": 0.048628050833940506,
      "rewards/rejected": -0.010045038536190987,
      "step": 1030
    },
    {
      "epoch": 0.77,
      "grad_norm": 0.484375,
      "learning_rate": 7.934130937159508e-08,
      "logits/chosen": 0.0004897325998172164,
      "logits/rejected": 0.7238715887069702,
      "logps/chosen": -279.21624755859375,
      "logps/rejected": -248.09957885742188,
      "loss": 0.6519,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.07424648851156235,
      "rewards/margins": 0.08449728786945343,
      "rewards/margins_max": 0.1293594241142273,
      "rewards/margins_min": 0.039635173976421356,
      "rewards/margins_std": 0.06344461441040039,
      "rewards/rejected": -0.010250799357891083,
      "step": 1040
    },
    {
      "epoch": 0.77,
      "grad_norm": 0.455078125,
      "learning_rate": 7.470526246864364e-08,
      "logits/chosen": -0.028243955224752426,
      "logits/rejected": 0.6930115818977356,
      "logps/chosen": -301.51458740234375,
      "logps/rejected": -251.8220672607422,
      "loss": 0.6559,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06157411262392998,
      "rewards/margins": 0.07462558895349503,
      "rewards/margins_max": 0.10729455947875977,
      "rewards/margins_min": 0.04195661470293999,
      "rewards/margins_std": 0.04620090872049332,
      "rewards/rejected": -0.01305147260427475,
      "step": 1050
    },
    {
      "epoch": 0.78,
      "grad_norm": 0.494140625,
      "learning_rate": 7.018487786691512e-08,
      "logits/chosen": -0.18514272570610046,
      "logits/rejected": 0.8259115219116211,
      "logps/chosen": -284.2102355957031,
      "logps/rejected": -236.52206420898438,
      "loss": 0.6474,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.09142851829528809,
      "rewards/margins": 0.10867790132761002,
      "rewards/margins_max": 0.16026386618614197,
      "rewards/margins_min": 0.05709192156791687,
      "rewards/margins_std": 0.07295358926057816,
      "rewards/rejected": -0.01724938116967678,
      "step": 1060
    },
    {
      "epoch": 0.79,
      "grad_norm": 0.4296875,
      "learning_rate": 6.578313818923559e-08,
      "logits/chosen": 0.12218723446130753,
      "logits/rejected": 0.6379785537719727,
      "logps/chosen": -238.366943359375,
      "logps/rejected": -233.61026000976562,
      "loss": 0.6535,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.055001966655254364,
      "rewards/margins": 0.06915679574012756,
      "rewards/margins_max": 0.09845878183841705,
      "rewards/margins_min": 0.039854805916547775,
      "rewards/margins_std": 0.04143926501274109,
      "rewards/rejected": -0.014154826290905476,
      "step": 1070
    },
    {
      "epoch": 0.79,
      "grad_norm": 0.40234375,
      "learning_rate": 6.15029477745925e-08,
      "logits/chosen": -0.1947169005870819,
      "logits/rejected": 0.7179273366928101,
      "logps/chosen": -295.55096435546875,
      "logps/rejected": -222.1858367919922,
      "loss": 0.6512,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07841251790523529,
      "rewards/margins": 0.08169189840555191,
      "rewards/margins_max": 0.11758895218372345,
      "rewards/margins_min": 0.045794855803251266,
      "rewards/margins_std": 0.05076608806848526,
      "rewards/rejected": -0.003279387950897217,
      "step": 1080
    },
    {
      "epoch": 0.8,
      "grad_norm": 0.44921875,
      "learning_rate": 5.734713076180486e-08,
      "logits/chosen": 0.006981179118156433,
      "logits/rejected": 0.7452491521835327,
      "logps/chosen": -260.1736145019531,
      "logps/rejected": -224.355224609375,
      "loss": 0.6557,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.06287099421024323,
      "rewards/margins": 0.07665695995092392,
      "rewards/margins_max": 0.11319824308156967,
      "rewards/margins_min": 0.04011568799614906,
      "rewards/margins_std": 0.05167717486619949,
      "rewards/rejected": -0.013785977847874165,
      "step": 1090
    },
    {
      "epoch": 0.81,
      "grad_norm": 0.48046875,
      "learning_rate": 5.3318429226110875e-08,
      "logits/chosen": 0.015549438074231148,
      "logits/rejected": 0.6923533082008362,
      "logps/chosen": -269.09747314453125,
      "logps/rejected": -236.7939910888672,
      "loss": 0.6512,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06426068395376205,
      "rewards/margins": 0.08864103257656097,
      "rewards/margins_max": 0.12798473238945007,
      "rewards/margins_min": 0.04929735139012337,
      "rewards/margins_std": 0.05564037710428238,
      "rewards/rejected": -0.024380357936024666,
      "step": 1100
    },
    {
      "epoch": 0.82,
      "grad_norm": 0.419921875,
      "learning_rate": 4.9419501369902026e-08,
      "logits/chosen": -0.029879633337259293,
      "logits/rejected": 0.7326146960258484,
      "logps/chosen": -260.98968505859375,
      "logps/rejected": -220.9407958984375,
      "loss": 0.6499,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07826592028141022,
      "rewards/margins": 0.09226210415363312,
      "rewards/margins_max": 0.1359224021434784,
      "rewards/margins_min": 0.04860180616378784,
      "rewards/margins_std": 0.06174498051404953,
      "rewards/rejected": -0.0139961838722229,
      "step": 1110
    },
    {
      "epoch": 0.82,
      "grad_norm": 0.42578125,
      "learning_rate": 4.5652919768798896e-08,
      "logits/chosen": -0.1553410142660141,
      "logits/rejected": 0.6449904441833496,
      "logps/chosen": -280.3263244628906,
      "logps/rejected": -242.4776153564453,
      "loss": 0.6512,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.074976846575737,
      "rewards/margins": 0.08818279206752777,
      "rewards/margins_max": 0.12042693048715591,
      "rewards/margins_min": 0.05593867227435112,
      "rewards/margins_std": 0.04560008645057678,
      "rewards/rejected": -0.013205950148403645,
      "step": 1120
    },
    {
      "epoch": 0.83,
      "grad_norm": 0.474609375,
      "learning_rate": 4.2021169674223536e-08,
      "logits/chosen": -0.12581007182598114,
      "logits/rejected": 0.6627474427223206,
      "logps/chosen": -274.0786437988281,
      "logps/rejected": -250.4473114013672,
      "loss": 0.6483,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.0676179900765419,
      "rewards/margins": 0.08669795840978622,
      "rewards/margins_max": 0.12913702428340912,
      "rewards/margins_min": 0.04425888508558273,
      "rewards/margins_std": 0.060017913579940796,
      "rewards/rejected": -0.01907997578382492,
      "step": 1130
    },
    {
      "epoch": 0.84,
      "grad_norm": 0.369140625,
      "learning_rate": 3.852664737359046e-08,
      "logits/chosen": 0.08468042314052582,
      "logits/rejected": 0.7667611241340637,
      "logps/chosen": -249.0133819580078,
      "logps/rejected": -232.16561889648438,
      "loss": 0.6497,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.07480134069919586,
      "rewards/margins": 0.09523622691631317,
      "rewards/margins_max": 0.15331092476844788,
      "rewards/margins_min": 0.037161558866500854,
      "rewards/margins_std": 0.08212999999523163,
      "rewards/rejected": -0.020434897392988205,
      "step": 1140
    },
    {
      "epoch": 0.85,
      "grad_norm": 0.373046875,
      "learning_rate": 3.5171658609197824e-08,
      "logits/chosen": -0.03485478088259697,
      "logits/rejected": 0.5982181429862976,
      "logps/chosen": -252.3005828857422,
      "logps/rejected": -240.3730926513672,
      "loss": 0.6515,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06789231300354004,
      "rewards/margins": 0.0819520652294159,
      "rewards/margins_max": 0.11909864842891693,
      "rewards/margins_min": 0.04480547830462456,
      "rewards/margins_std": 0.05253320187330246,
      "rewards/rejected": -0.014059755019843578,
      "step": 1150
    },
    {
      "epoch": 0.85,
      "grad_norm": 0.435546875,
      "learning_rate": 3.195841705686139e-08,
      "logits/chosen": 0.001758894301019609,
      "logits/rejected": 0.6828774213790894,
      "logps/chosen": -259.75640869140625,
      "logps/rejected": -243.09970092773438,
      "loss": 0.6517,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.06515143811702728,
      "rewards/margins": 0.07973488420248032,
      "rewards/margins_max": 0.11704728752374649,
      "rewards/margins_min": 0.042422495782375336,
      "rewards/margins_std": 0.05276770144701004,
      "rewards/rejected": -0.014583444222807884,
      "step": 1160
    },
    {
      "epoch": 0.86,
      "grad_norm": 0.4375,
      "learning_rate": 2.8889042865294837e-08,
      "logits/chosen": 0.0009852380026131868,
      "logits/rejected": 0.7770395278930664,
      "logps/chosen": -252.7977752685547,
      "logps/rejected": -211.03494262695312,
      "loss": 0.652,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.06879086047410965,
      "rewards/margins": 0.0823785737156868,
      "rewards/margins_max": 0.126399964094162,
      "rewards/margins_min": 0.038357168436050415,
      "rewards/margins_std": 0.06225566938519478,
      "rewards/rejected": -0.013587711378932,
      "step": 1170
    },
    {
      "epoch": 0.87,
      "grad_norm": 0.5234375,
      "learning_rate": 2.5965561257202036e-08,
      "logits/chosen": -0.0783575028181076,
      "logits/rejected": 0.7545520067214966,
      "logps/chosen": -272.24041748046875,
      "logps/rejected": -247.2455291748047,
      "loss": 0.6473,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.09078168123960495,
      "rewards/margins": 0.10268386453390121,
      "rewards/margins_max": 0.14412382245063782,
      "rewards/margins_min": 0.06124391406774521,
      "rewards/margins_std": 0.05860493704676628,
      "rewards/rejected": -0.011902175843715668,
      "step": 1180
    },
    {
      "epoch": 0.88,
      "grad_norm": 0.80078125,
      "learning_rate": 2.318990119300218e-08,
      "logits/chosen": 0.05255628377199173,
      "logits/rejected": 0.6947780847549438,
      "logps/chosen": -243.50509643554688,
      "logps/rejected": -212.8872528076172,
      "loss": 0.6546,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05941227823495865,
      "rewards/margins": 0.07317686825990677,
      "rewards/margins_max": 0.11058169603347778,
      "rewards/margins_min": 0.03577204793691635,
      "rewards/margins_std": 0.052898406982421875,
      "rewards/rejected": -0.013764592818915844,
      "step": 1190
    },
    {
      "epoch": 0.88,
      "grad_norm": 0.4296875,
      "learning_rate": 2.0563894098070216e-08,
      "logits/chosen": -0.0692586898803711,
      "logits/rejected": 0.7870053052902222,
      "logps/chosen": -309.2232666015625,
      "logps/rejected": -266.5531921386719,
      "loss": 0.6476,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": 0.08217885345220566,
      "rewards/margins": 0.09460541605949402,
      "rewards/margins_max": 0.13726098835468292,
      "rewards/margins_min": 0.05194983631372452,
      "rewards/margins_std": 0.06032410264015198,
      "rewards/rejected": -0.012426558881998062,
      "step": 1200
    },
    {
      "epoch": 0.89,
      "grad_norm": 0.34375,
      "learning_rate": 1.8089272654333353e-08,
      "logits/chosen": -0.046720363199710846,
      "logits/rejected": 0.6504586338996887,
      "logps/chosen": -261.49298095703125,
      "logps/rejected": -223.7759552001953,
      "loss": 0.6526,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07228533923625946,
      "rewards/margins": 0.07905267179012299,
      "rewards/margins_max": 0.12283986806869507,
      "rewards/margins_min": 0.035265471786260605,
      "rewards/margins_std": 0.06192445755004883,
      "rewards/rejected": -0.006767333950847387,
      "step": 1210
    },
    {
      "epoch": 0.9,
      "grad_norm": 0.4140625,
      "learning_rate": 1.5767669657019005e-08,
      "logits/chosen": -0.1003817468881607,
      "logits/rejected": 0.7918148636817932,
      "logps/chosen": -284.4376525878906,
      "logps/rejected": -226.739990234375,
      "loss": 0.6514,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07430224120616913,
      "rewards/margins": 0.08967499434947968,
      "rewards/margins_max": 0.13941441476345062,
      "rewards/margins_min": 0.03993559628725052,
      "rewards/margins_std": 0.07034213840961456,
      "rewards/rejected": -0.015372763387858868,
      "step": 1220
    },
    {
      "epoch": 0.91,
      "grad_norm": 0.42578125,
      "learning_rate": 1.3600616937310267e-08,
      "logits/chosen": -0.13235566020011902,
      "logits/rejected": 0.602156400680542,
      "logps/chosen": -261.45050048828125,
      "logps/rejected": -231.56277465820312,
      "loss": 0.647,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.0806741714477539,
      "rewards/margins": 0.10508724302053452,
      "rewards/margins_max": 0.1610412895679474,
      "rewards/margins_min": 0.049133218824863434,
      "rewards/margins_std": 0.07913094758987427,
      "rewards/rejected": -0.024413082748651505,
      "step": 1230
    },
    {
      "epoch": 0.91,
      "grad_norm": 0.478515625,
      "learning_rate": 1.1589544351619047e-08,
      "logits/chosen": 0.02703939750790596,
      "logits/rejected": 0.8025895357131958,
      "logps/chosen": -334.37176513671875,
      "logps/rejected": -263.4154968261719,
      "loss": 0.6488,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07323239743709564,
      "rewards/margins": 0.08194652944803238,
      "rewards/margins_max": 0.12296142429113388,
      "rewards/margins_min": 0.04093163460493088,
      "rewards/margins_std": 0.05800382047891617,
      "rewards/rejected": -0.008714130148291588,
      "step": 1240
    },
    {
      "epoch": 0.92,
      "grad_norm": 0.359375,
      "learning_rate": 9.735778838143749e-09,
      "logits/chosen": -0.0286453515291214,
      "logits/rejected": 0.7510533332824707,
      "logps/chosen": -311.01959228515625,
      "logps/rejected": -240.82870483398438,
      "loss": 0.6492,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.07665716856718063,
      "rewards/margins": 0.09343422949314117,
      "rewards/margins_max": 0.13625505566596985,
      "rewards/margins_min": 0.0506133958697319,
      "rewards/margins_std": 0.06055780500173569,
      "rewards/rejected": -0.01677705906331539,
      "step": 1250
    },
    {
      "epoch": 0.93,
      "grad_norm": 0.44140625,
      "learning_rate": 8.040543541333655e-09,
      "logits/chosen": 0.05244514346122742,
      "logits/rejected": 0.7751520276069641,
      "logps/chosen": -310.7460021972656,
      "logps/rejected": -286.03826904296875,
      "loss": 0.6489,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": 0.06985687464475632,
      "rewards/margins": 0.0900772362947464,
      "rewards/margins_max": 0.12786877155303955,
      "rewards/margins_min": 0.052285682410001755,
      "rewards/margins_std": 0.053445328027009964,
      "rewards/rejected": -0.020220357924699783,
      "step": 1260
    },
    {
      "epoch": 0.93,
      "grad_norm": 0.5,
      "learning_rate": 6.504957004838746e-09,
      "logits/chosen": -0.03165416046977043,
      "logits/rejected": 0.822761058807373,
      "logps/chosen": -310.04400634765625,
      "logps/rejected": -237.48782348632812,
      "loss": 0.6523,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06924634426832199,
      "rewards/margins": 0.08929922431707382,
      "rewards/margins_max": 0.14400045573711395,
      "rewards/margins_min": 0.034597981721162796,
      "rewards/margins_std": 0.07735923677682877,
      "rewards/rejected": -0.020052876323461533,
      "step": 1270
    },
    {
      "epoch": 0.94,
      "grad_norm": 0.4765625,
      "learning_rate": 5.130032433476483e-09,
      "logits/chosen": -0.011189639568328857,
      "logits/rejected": 0.691806972026825,
      "logps/chosen": -271.3976135253906,
      "logps/rejected": -269.19390869140625,
      "loss": 0.6545,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.08424172550439835,
      "rewards/margins": 0.09692950546741486,
      "rewards/margins_max": 0.14971934258937836,
      "rewards/margins_min": 0.044139645993709564,
      "rewards/margins_std": 0.07465612888336182,
      "rewards/rejected": -0.012687774375081062,
      "step": 1280
    },
    {
      "epoch": 0.95,
      "grad_norm": 0.43359375,
      "learning_rate": 3.916677024702858e-09,
      "logits/chosen": 0.009526084177196026,
      "logits/rejected": 0.7200338244438171,
      "logps/chosen": -255.1676788330078,
      "logps/rejected": -234.27432250976562,
      "loss": 0.6534,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06782687455415726,
      "rewards/margins": 0.08498277515172958,
      "rewards/margins_max": 0.1279430389404297,
      "rewards/margins_min": 0.04202253371477127,
      "rewards/margins_std": 0.06075497344136238,
      "rewards/rejected": -0.017155908048152924,
      "step": 1290
    },
    {
      "epoch": 0.96,
      "grad_norm": 0.51953125,
      "learning_rate": 2.865691370028761e-09,
      "logits/chosen": -0.11016593128442764,
      "logits/rejected": 0.7786885499954224,
      "logps/chosen": -310.4071960449219,
      "logps/rejected": -242.39505004882812,
      "loss": 0.6487,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0797882154583931,
      "rewards/margins": 0.09522275626659393,
      "rewards/margins_max": 0.13675549626350403,
      "rewards/margins_min": 0.05369000881910324,
      "rewards/margins_std": 0.0587361678481102,
      "rewards/rejected": -0.015434542670845985,
      "step": 1300
    },
    {
      "epoch": 0.96,
      "grad_norm": 0.482421875,
      "learning_rate": 1.977768926776896e-09,
      "logits/chosen": 0.07193834334611893,
      "logits/rejected": 0.7082736492156982,
      "logps/chosen": -269.1424255371094,
      "logps/rejected": -258.5067443847656,
      "loss": 0.6549,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": 0.07826878130435944,
      "rewards/margins": 0.0938304141163826,
      "rewards/margins_max": 0.13868080079555511,
      "rewards/margins_min": 0.04898003116250038,
      "rewards/margins_std": 0.06342802941799164,
      "rewards/rejected": -0.01556164026260376,
      "step": 1310
    },
    {
      "epoch": 0.97,
      "grad_norm": 0.474609375,
      "learning_rate": 1.2534955605274233e-09,
      "logits/chosen": -0.014053803868591785,
      "logits/rejected": 0.777262806892395,
      "logps/chosen": -242.13796997070312,
      "logps/rejected": -218.4778594970703,
      "loss": 0.6505,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.07983841001987457,
      "rewards/margins": 0.09085123986005783,
      "rewards/margins_max": 0.1351737529039383,
      "rewards/margins_min": 0.046528737992048264,
      "rewards/margins_std": 0.06268148869276047,
      "rewards/rejected": -0.011012842878699303,
      "step": 1320
    },
    {
      "epoch": 0.98,
      "grad_norm": 0.48828125,
      "learning_rate": 6.933491585542351e-10,
      "logits/chosen": 0.03265099227428436,
      "logits/rejected": 0.7704384326934814,
      "logps/chosen": -293.1032409667969,
      "logps/rejected": -263.2946472167969,
      "loss": 0.6471,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.08791947364807129,
      "rewards/margins": 0.10431689023971558,
      "rewards/margins_max": 0.15510039031505585,
      "rewards/margins_min": 0.05353338643908501,
      "rewards/margins_std": 0.07181872427463531,
      "rewards/rejected": -0.016397414728999138,
      "step": 1330
    },
    {
      "epoch": 0.99,
      "grad_norm": 0.451171875,
      "learning_rate": 2.9769931450737694e-10,
      "logits/chosen": 0.03406764939427376,
      "logits/rejected": 0.7593650221824646,
      "logps/chosen": -301.3545227050781,
      "logps/rejected": -253.1154022216797,
      "loss": 0.6492,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": 0.06837470084428787,
      "rewards/margins": 0.08594924956560135,
      "rewards/margins_max": 0.1240011602640152,
      "rewards/margins_min": 0.0478973314166069,
      "rewards/margins_std": 0.05381352826952934,
      "rewards/rejected": -0.017574548721313477,
      "step": 1340
    },
    {
      "epoch": 0.99,
      "grad_norm": 0.412109375,
      "learning_rate": 6.680708454906425e-11,
      "logits/chosen": -0.05887039750814438,
      "logits/rejected": 0.7168238759040833,
      "logps/chosen": -274.7958679199219,
      "logps/rejected": -234.88424682617188,
      "loss": 0.6506,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": 0.06663821637630463,
      "rewards/margins": 0.08757266402244568,
      "rewards/margins_max": 0.1257934719324112,
      "rewards/margins_min": 0.049351852387189865,
      "rewards/margins_std": 0.05405237525701523,
      "rewards/rejected": -0.020934434607625008,
      "step": 1350
    },
    {
      "epoch": 1.0,
      "eval_logits/chosen": 0.744699239730835,
      "eval_logits/rejected": 0.8899587988853455,
      "eval_logps/chosen": -330.823974609375,
      "eval_logps/rejected": -316.1527404785156,
      "eval_loss": 0.6946464776992798,
      "eval_rewards/accuracies": 0.5195000171661377,
      "eval_rewards/chosen": 0.03155522421002388,
      "eval_rewards/margins": -0.0017884434200823307,
      "eval_rewards/margins_max": 0.09517286717891693,
      "eval_rewards/margins_min": -0.10408221185207367,
      "eval_rewards/margins_std": 0.06461790204048157,
      "eval_rewards/rejected": 0.03334366902709007,
      "eval_runtime": 837.0887,
      "eval_samples_per_second": 4.778,
      "eval_steps_per_second": 0.299,
      "step": 1359
    },
    {
      "epoch": 1.0,
      "step": 1359,
      "total_flos": 0.0,
      "train_loss": 0.6604200788417925,
      "train_runtime": 10358.1715,
      "train_samples_per_second": 2.099,
      "train_steps_per_second": 0.131
    }
  ],
  "logging_steps": 10,
  "max_steps": 1359,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}