{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 64,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": -1.0596340894699097,
      "debug/policy_chosen_logps": -179.04273986816406,
      "debug/policy_rejected_logits": -1.1748394966125488,
      "debug/policy_rejected_logps": -295.01690673828125,
      "debug/reference_chosen_logps": -179.04273986816406,
      "debug/reference_rejected_logps": -295.01690673828125,
      "epoch": 0.015625,
      "grad_norm": 52.30319105460711,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0596340894699097,
      "logits/rejected": -1.1748394966125488,
      "logps/chosen": -179.04273986816406,
      "logps/rejected": -295.01690673828125,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": -1.1150486469268799,
      "debug/policy_chosen_logps": -124.63790893554688,
      "debug/policy_rejected_logits": -1.0623761415481567,
      "debug/policy_rejected_logps": -270.75244140625,
      "debug/reference_chosen_logps": -125.14633178710938,
      "debug/reference_rejected_logps": -271.20208740234375,
      "epoch": 0.03125,
      "grad_norm": 29.130704023833047,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1150486469268799,
      "logits/rejected": -1.0623761415481567,
      "logps/chosen": -124.63790893554688,
      "logps/rejected": -270.75244140625,
      "loss": 0.4989,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.005084190517663956,
      "rewards/margins": 0.0005879019154235721,
      "rewards/rejected": 0.0044962880201637745,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": -1.1071562767028809,
      "debug/policy_chosen_logps": -136.3170166015625,
      "debug/policy_rejected_logits": -1.1613606214523315,
      "debug/policy_rejected_logps": -268.709228515625,
      "debug/reference_chosen_logps": -137.68783569335938,
      "debug/reference_rejected_logps": -268.8507995605469,
      "epoch": 0.046875,
      "grad_norm": 24.965184935253273,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1071562767028809,
      "logits/rejected": -1.1613606214523315,
      "logps/chosen": -136.3170166015625,
      "logps/rejected": -268.709228515625,
      "loss": 0.4914,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.013708190061151981,
      "rewards/margins": 0.012292098253965378,
      "rewards/rejected": 0.0014160918071866035,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": -1.066061019897461,
      "debug/policy_chosen_logps": -153.8428192138672,
      "debug/policy_rejected_logits": -1.1866570711135864,
      "debug/policy_rejected_logps": -274.9277648925781,
      "debug/reference_chosen_logps": -155.69000244140625,
      "debug/reference_rejected_logps": -275.12884521484375,
      "epoch": 0.0625,
      "grad_norm": 24.49810670915077,
      "learning_rate": 1e-06,
      "logits/chosen": -1.066061019897461,
      "logits/rejected": -1.1866570711135864,
      "logps/chosen": -153.8428192138672,
      "logps/rejected": -274.9277648925781,
      "loss": 0.4777,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.018471689894795418,
      "rewards/margins": 0.016461096704006195,
      "rewards/rejected": 0.0020105931907892227,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": -1.0783909559249878,
      "debug/policy_chosen_logps": -161.8551483154297,
      "debug/policy_rejected_logits": -1.1809625625610352,
      "debug/policy_rejected_logps": -291.5763244628906,
      "debug/reference_chosen_logps": -165.77706909179688,
      "debug/reference_rejected_logps": -290.215087890625,
      "epoch": 0.078125,
      "grad_norm": 22.066344534464825,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0783909559249878,
      "logits/rejected": -1.1809625625610352,
      "logps/chosen": -161.8551483154297,
      "logps/rejected": -291.5763244628906,
      "loss": 0.4425,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.039219196885824203,
      "rewards/margins": 0.05283135548233986,
      "rewards/rejected": -0.013612156733870506,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": -1.0005463361740112,
      "debug/policy_chosen_logps": -177.85003662109375,
      "debug/policy_rejected_logits": -1.0288403034210205,
      "debug/policy_rejected_logps": -263.21014404296875,
      "debug/reference_chosen_logps": -178.246337890625,
      "debug/reference_rejected_logps": -263.5099182128906,
      "epoch": 0.09375,
      "grad_norm": 41.16778948079108,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0005463361740112,
      "logits/rejected": -1.0288403034210205,
      "logps/chosen": -177.85003662109375,
      "logps/rejected": -263.21014404296875,
      "loss": 0.4659,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.003962935879826546,
      "rewards/margins": 0.0009648129343986511,
      "rewards/rejected": 0.0029981210827827454,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": -0.9317433834075928,
      "debug/policy_chosen_logps": -155.7017822265625,
      "debug/policy_rejected_logits": -1.3209773302078247,
      "debug/policy_rejected_logps": -308.2155456542969,
      "debug/reference_chosen_logps": -165.14569091796875,
      "debug/reference_rejected_logps": -295.0081481933594,
      "epoch": 0.109375,
      "grad_norm": 15.904262612549944,
      "learning_rate": 1e-06,
      "logits/chosen": -0.9317433834075928,
      "logits/rejected": -1.3209773302078247,
      "logps/chosen": -155.7017822265625,
      "logps/rejected": -308.2155456542969,
      "loss": 0.4323,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0944390594959259,
      "rewards/margins": 0.22651299834251404,
      "rewards/rejected": -0.13207395374774933,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": -1.0539729595184326,
      "debug/policy_chosen_logps": -173.66781616210938,
      "debug/policy_rejected_logits": -1.0206472873687744,
      "debug/policy_rejected_logps": -271.9588317871094,
      "debug/reference_chosen_logps": -177.30899047851562,
      "debug/reference_rejected_logps": -263.9579162597656,
      "epoch": 0.125,
      "grad_norm": 15.028091497342194,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0539729595184326,
      "logits/rejected": -1.0206472873687744,
      "logps/chosen": -173.66781616210938,
      "logps/rejected": -271.9588317871094,
      "loss": 0.4255,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.036411646753549576,
      "rewards/margins": 0.1164209246635437,
      "rewards/rejected": -0.08000928163528442,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": -0.9866081476211548,
      "debug/policy_chosen_logps": -176.56866455078125,
      "debug/policy_rejected_logits": -0.9740838408470154,
      "debug/policy_rejected_logps": -272.35650634765625,
      "debug/reference_chosen_logps": -177.0741729736328,
      "debug/reference_rejected_logps": -260.4818420410156,
      "epoch": 0.140625,
      "grad_norm": 32.8906220838234,
      "learning_rate": 1e-06,
      "logits/chosen": -0.9866081476211548,
      "logits/rejected": -0.9740838408470154,
      "logps/chosen": -176.56866455078125,
      "logps/rejected": -272.35650634765625,
      "loss": 0.4107,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.005055226851254702,
      "rewards/margins": 0.12380212545394897,
      "rewards/rejected": -0.11874689161777496,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": -0.9441277980804443,
      "debug/policy_chosen_logps": -164.24789428710938,
      "debug/policy_rejected_logits": -1.1364271640777588,
      "debug/policy_rejected_logps": -292.0938720703125,
      "debug/reference_chosen_logps": -160.7564697265625,
      "debug/reference_rejected_logps": -257.1752014160156,
      "epoch": 0.15625,
      "grad_norm": 28.811843166780264,
      "learning_rate": 1e-06,
      "logits/chosen": -0.9441277980804443,
      "logits/rejected": -1.1364271640777588,
      "logps/chosen": -164.24789428710938,
      "logps/rejected": -292.0938720703125,
      "loss": 0.4237,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03491419926285744,
      "rewards/margins": 0.3142724931240082,
      "rewards/rejected": -0.3491867184638977,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": -0.9175143837928772,
      "debug/policy_chosen_logps": -214.65664672851562,
      "debug/policy_rejected_logits": -1.1515822410583496,
      "debug/policy_rejected_logps": -244.6530303955078,
      "debug/reference_chosen_logps": -207.79930114746094,
      "debug/reference_rejected_logps": -230.90333557128906,
      "epoch": 0.171875,
      "grad_norm": 38.46428758925275,
      "learning_rate": 1e-06,
      "logits/chosen": -0.9175143837928772,
      "logits/rejected": -1.1515822410583496,
      "logps/chosen": -214.65664672851562,
      "logps/rejected": -244.6530303955078,
      "loss": 0.4949,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.06857340782880783,
      "rewards/margins": 0.06892354786396027,
      "rewards/rejected": -0.1374969631433487,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": -0.8965519070625305,
      "debug/policy_chosen_logps": -153.26284790039062,
      "debug/policy_rejected_logits": -1.1321805715560913,
      "debug/policy_rejected_logps": -318.78076171875,
      "debug/reference_chosen_logps": -154.14707946777344,
      "debug/reference_rejected_logps": -289.067138671875,
      "epoch": 0.1875,
      "grad_norm": 41.433140559474445,
      "learning_rate": 1e-06,
      "logits/chosen": -0.8965519070625305,
      "logits/rejected": -1.1321805715560913,
      "logps/chosen": -153.26284790039062,
      "logps/rejected": -318.78076171875,
      "loss": 0.4907,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.008842326700687408,
      "rewards/margins": 0.30597835779190063,
      "rewards/rejected": -0.2971360683441162,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": -0.945601761341095,
      "debug/policy_chosen_logps": -122.90229797363281,
      "debug/policy_rejected_logits": -1.0716924667358398,
      "debug/policy_rejected_logps": -274.2931823730469,
      "debug/reference_chosen_logps": -120.32145690917969,
      "debug/reference_rejected_logps": -250.55557250976562,
      "epoch": 0.203125,
      "grad_norm": 26.79881614435138,
      "learning_rate": 1e-06,
      "logits/chosen": -0.945601761341095,
      "logits/rejected": -1.0716924667358398,
      "logps/chosen": -122.90229797363281,
      "logps/rejected": -274.2931823730469,
      "loss": 0.4694,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.02580837532877922,
      "rewards/margins": 0.2115677297115326,
      "rewards/rejected": -0.2373761087656021,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": -1.0047388076782227,
      "debug/policy_chosen_logps": -200.4830780029297,
      "debug/policy_rejected_logits": -1.1980981826782227,
      "debug/policy_rejected_logps": -315.792236328125,
      "debug/reference_chosen_logps": -190.80075073242188,
      "debug/reference_rejected_logps": -281.5347595214844,
      "epoch": 0.21875,
      "grad_norm": 27.316365360407435,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0047388076782227,
      "logits/rejected": -1.1980981826782227,
      "logps/chosen": -200.4830780029297,
      "logps/rejected": -315.792236328125,
      "loss": 0.4324,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.09682333469390869,
      "rewards/margins": 0.24575121700763702,
      "rewards/rejected": -0.3425745368003845,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": -1.1348706483840942,
      "debug/policy_chosen_logps": -208.73074340820312,
      "debug/policy_rejected_logits": -1.121549129486084,
      "debug/policy_rejected_logps": -310.7353210449219,
      "debug/reference_chosen_logps": -204.0843048095703,
      "debug/reference_rejected_logps": -281.996337890625,
      "epoch": 0.234375,
      "grad_norm": 58.18504208169894,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1348706483840942,
      "logits/rejected": -1.121549129486084,
      "logps/chosen": -208.73074340820312,
      "logps/rejected": -310.7353210449219,
      "loss": 0.4662,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.04646441712975502,
      "rewards/margins": 0.24092541635036469,
      "rewards/rejected": -0.2873898148536682,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": -0.9974825978279114,
      "debug/policy_chosen_logps": -154.0273895263672,
      "debug/policy_rejected_logits": -1.1503194570541382,
      "debug/policy_rejected_logps": -307.7276611328125,
      "debug/reference_chosen_logps": -154.69586181640625,
      "debug/reference_rejected_logps": -273.1531677246094,
      "epoch": 0.25,
      "grad_norm": 56.48600158612175,
      "learning_rate": 1e-06,
      "logits/chosen": -0.9974825978279114,
      "logits/rejected": -1.1503194570541382,
      "logps/chosen": -154.0273895263672,
      "logps/rejected": -307.7276611328125,
      "loss": 0.4093,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.006684892810881138,
      "rewards/margins": 0.3524298071861267,
      "rewards/rejected": -0.34574490785598755,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": -1.0567247867584229,
      "debug/policy_chosen_logps": -137.61720275878906,
      "debug/policy_rejected_logits": -1.0961592197418213,
      "debug/policy_rejected_logps": -313.12060546875,
      "debug/reference_chosen_logps": -135.6652069091797,
      "debug/reference_rejected_logps": -297.18695068359375,
      "epoch": 0.265625,
      "grad_norm": 31.458528575785774,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0567247867584229,
      "logits/rejected": -1.0961592197418213,
      "logps/chosen": -137.61720275878906,
      "logps/rejected": -313.12060546875,
      "loss": 0.4473,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.01951989158987999,
      "rewards/margins": 0.13981682062149048,
      "rewards/rejected": -0.15933671593666077,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": -1.0550764799118042,
      "debug/policy_chosen_logps": -143.5434112548828,
      "debug/policy_rejected_logits": -1.3183400630950928,
      "debug/policy_rejected_logps": -359.35418701171875,
      "debug/reference_chosen_logps": -157.90188598632812,
      "debug/reference_rejected_logps": -317.474853515625,
      "epoch": 0.28125,
      "grad_norm": 24.003283570475016,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0550764799118042,
      "logits/rejected": -1.3183400630950928,
      "logps/chosen": -143.5434112548828,
      "logps/rejected": -359.35418701171875,
      "loss": 0.4381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14358465373516083,
      "rewards/margins": 0.5623779296875,
      "rewards/rejected": -0.418793261051178,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": -1.1249719858169556,
      "debug/policy_chosen_logps": -163.00375366210938,
      "debug/policy_rejected_logits": -1.1258165836334229,
      "debug/policy_rejected_logps": -283.2430725097656,
      "debug/reference_chosen_logps": -166.72418212890625,
      "debug/reference_rejected_logps": -264.2232360839844,
      "epoch": 0.296875,
      "grad_norm": 41.98096605753313,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1249719858169556,
      "logits/rejected": -1.1258165836334229,
      "logps/chosen": -163.00375366210938,
      "logps/rejected": -283.2430725097656,
      "loss": 0.4597,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03720443695783615,
      "rewards/margins": 0.22740286588668823,
      "rewards/rejected": -0.19019843637943268,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": -1.000652551651001,
      "debug/policy_chosen_logps": -174.8540802001953,
      "debug/policy_rejected_logits": -1.075732946395874,
      "debug/policy_rejected_logps": -248.3970947265625,
      "debug/reference_chosen_logps": -179.03424072265625,
      "debug/reference_rejected_logps": -235.50778198242188,
      "epoch": 0.3125,
      "grad_norm": 26.892461198324778,
      "learning_rate": 1e-06,
      "logits/chosen": -1.000652551651001,
      "logits/rejected": -1.075732946395874,
      "logps/chosen": -174.8540802001953,
      "logps/rejected": -248.3970947265625,
      "loss": 0.4325,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.041801512241363525,
      "rewards/margins": 0.1706947386264801,
      "rewards/rejected": -0.12889322638511658,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": -1.0587340593338013,
      "debug/policy_chosen_logps": -148.18423461914062,
      "debug/policy_rejected_logits": -1.435739278793335,
      "debug/policy_rejected_logps": -331.1427001953125,
      "debug/reference_chosen_logps": -151.2082061767578,
      "debug/reference_rejected_logps": -314.77117919921875,
      "epoch": 0.328125,
      "grad_norm": 15.800648562809261,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0587340593338013,
      "logits/rejected": -1.435739278793335,
      "logps/chosen": -148.18423461914062,
      "logps/rejected": -331.1427001953125,
      "loss": 0.3982,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.030239801853895187,
      "rewards/margins": 0.1939551830291748,
      "rewards/rejected": -0.16371536254882812,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": -1.0205200910568237,
      "debug/policy_chosen_logps": -157.31350708007812,
      "debug/policy_rejected_logits": -1.0888888835906982,
      "debug/policy_rejected_logps": -346.0768127441406,
      "debug/reference_chosen_logps": -161.5574493408203,
      "debug/reference_rejected_logps": -338.91650390625,
      "epoch": 0.34375,
      "grad_norm": 21.49065797596958,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0205200910568237,
      "logits/rejected": -1.0888888835906982,
      "logps/chosen": -157.31350708007812,
      "logps/rejected": -346.0768127441406,
      "loss": 0.4361,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.042439430952072144,
      "rewards/margins": 0.11404269933700562,
      "rewards/rejected": -0.07160326838493347,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": -1.1462302207946777,
      "debug/policy_chosen_logps": -195.76788330078125,
      "debug/policy_rejected_logits": -1.2484185695648193,
      "debug/policy_rejected_logps": -277.576904296875,
      "debug/reference_chosen_logps": -198.74685668945312,
      "debug/reference_rejected_logps": -265.5393981933594,
      "epoch": 0.359375,
      "grad_norm": 17.749863549342045,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1462302207946777,
      "logits/rejected": -1.2484185695648193,
      "logps/chosen": -195.76788330078125,
      "logps/rejected": -277.576904296875,
      "loss": 0.4165,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.029789581894874573,
      "rewards/margins": 0.1501646637916565,
      "rewards/rejected": -0.12037509679794312,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": -0.952358067035675,
      "debug/policy_chosen_logps": -115.6708984375,
      "debug/policy_rejected_logits": -1.036898136138916,
      "debug/policy_rejected_logps": -245.47000122070312,
      "debug/reference_chosen_logps": -131.1976776123047,
      "debug/reference_rejected_logps": -238.638427734375,
      "epoch": 0.375,
      "grad_norm": 16.031924320507283,
      "learning_rate": 1e-06,
      "logits/chosen": -0.952358067035675,
      "logits/rejected": -1.036898136138916,
      "logps/chosen": -115.6708984375,
      "logps/rejected": -245.47000122070312,
      "loss": 0.3771,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1552678644657135,
      "rewards/margins": 0.22358371317386627,
      "rewards/rejected": -0.06831584870815277,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": -1.070897102355957,
      "debug/policy_chosen_logps": -178.87374877929688,
      "debug/policy_rejected_logits": -1.1623822450637817,
      "debug/policy_rejected_logps": -243.98184204101562,
      "debug/reference_chosen_logps": -179.05862426757812,
      "debug/reference_rejected_logps": -244.07818603515625,
      "epoch": 0.390625,
      "grad_norm": 38.66586744942012,
      "learning_rate": 1e-06,
      "logits/chosen": -1.070897102355957,
      "logits/rejected": -1.1623822450637817,
      "logps/chosen": -178.87374877929688,
      "logps/rejected": -243.98184204101562,
      "loss": 0.4396,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.001848660409450531,
      "rewards/margins": 0.0008851997554302216,
      "rewards/rejected": 0.000963456928730011,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": -1.1025017499923706,
      "debug/policy_chosen_logps": -173.5986328125,
      "debug/policy_rejected_logits": -1.1473654508590698,
      "debug/policy_rejected_logps": -245.47994995117188,
      "debug/reference_chosen_logps": -186.88778686523438,
      "debug/reference_rejected_logps": -241.27210998535156,
      "epoch": 0.40625,
      "grad_norm": 50.21384448251296,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1025017499923706,
      "logits/rejected": -1.1473654508590698,
      "logps/chosen": -173.5986328125,
      "logps/rejected": -245.47994995117188,
      "loss": 0.4023,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.13289162516593933,
      "rewards/margins": 0.1749698668718338,
      "rewards/rejected": -0.04207824170589447,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": -1.0175386667251587,
      "debug/policy_chosen_logps": -94.42786407470703,
      "debug/policy_rejected_logits": -1.1346431970596313,
      "debug/policy_rejected_logps": -240.36541748046875,
      "debug/reference_chosen_logps": -106.56871032714844,
      "debug/reference_rejected_logps": -240.29310607910156,
      "epoch": 0.421875,
      "grad_norm": 17.96088818186707,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0175386667251587,
      "logits/rejected": -1.1346431970596313,
      "logps/chosen": -94.42786407470703,
      "logps/rejected": -240.36541748046875,
      "loss": 0.39,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.12140841782093048,
      "rewards/margins": 0.12213139981031418,
      "rewards/rejected": -0.0007229708135128021,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": -1.1408073902130127,
      "debug/policy_chosen_logps": -126.60142517089844,
      "debug/policy_rejected_logits": -1.20956289768219,
      "debug/policy_rejected_logps": -313.8656311035156,
      "debug/reference_chosen_logps": -130.33799743652344,
      "debug/reference_rejected_logps": -291.3277893066406,
      "epoch": 0.4375,
      "grad_norm": 29.993745130410183,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1408073902130127,
      "logits/rejected": -1.20956289768219,
      "logps/chosen": -126.60142517089844,
      "logps/rejected": -313.8656311035156,
      "loss": 0.3947,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03736574202775955,
      "rewards/margins": 0.2627440094947815,
      "rewards/rejected": -0.22537828981876373,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": -1.0291798114776611,
      "debug/policy_chosen_logps": -192.92529296875,
      "debug/policy_rejected_logits": -1.2137432098388672,
      "debug/policy_rejected_logps": -315.1015930175781,
      "debug/reference_chosen_logps": -184.9921875,
      "debug/reference_rejected_logps": -301.6517639160156,
      "epoch": 0.453125,
      "grad_norm": 51.5929899248971,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0291798114776611,
      "logits/rejected": -1.2137432098388672,
      "logps/chosen": -192.92529296875,
      "logps/rejected": -315.1015930175781,
      "loss": 0.4253,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0793309286236763,
      "rewards/margins": 0.05516732484102249,
      "rewards/rejected": -0.1344982385635376,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": -1.033249020576477,
      "debug/policy_chosen_logps": -129.13734436035156,
      "debug/policy_rejected_logits": -1.1481682062149048,
      "debug/policy_rejected_logps": -319.0918884277344,
      "debug/reference_chosen_logps": -134.66598510742188,
      "debug/reference_rejected_logps": -297.1129150390625,
      "epoch": 0.46875,
      "grad_norm": 41.13041833853564,
      "learning_rate": 1e-06,
      "logits/chosen": -1.033249020576477,
      "logits/rejected": -1.1481682062149048,
      "logps/chosen": -129.13734436035156,
      "logps/rejected": -319.0918884277344,
      "loss": 0.4069,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05528645217418671,
      "rewards/margins": 0.2750762701034546,
      "rewards/rejected": -0.2197897881269455,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": -1.1428550481796265,
      "debug/policy_chosen_logps": -174.7340087890625,
      "debug/policy_rejected_logits": -1.017913818359375,
      "debug/policy_rejected_logps": -238.23471069335938,
      "debug/reference_chosen_logps": -180.0450897216797,
      "debug/reference_rejected_logps": -228.79031372070312,
      "epoch": 0.484375,
      "grad_norm": 54.8216481339695,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1428550481796265,
      "logits/rejected": -1.017913818359375,
      "logps/chosen": -174.7340087890625,
      "logps/rejected": -238.23471069335938,
      "loss": 0.4467,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.053110986948013306,
      "rewards/margins": 0.1475549191236496,
      "rewards/rejected": -0.0944439247250557,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": -1.0067996978759766,
      "debug/policy_chosen_logps": -145.49851989746094,
      "debug/policy_rejected_logits": -1.210583209991455,
      "debug/policy_rejected_logps": -274.90240478515625,
      "debug/reference_chosen_logps": -151.12542724609375,
      "debug/reference_rejected_logps": -264.36016845703125,
      "epoch": 0.5,
      "grad_norm": 31.943528016300796,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0067996978759766,
      "logits/rejected": -1.210583209991455,
      "logps/chosen": -145.49851989746094,
      "logps/rejected": -274.90240478515625,
      "loss": 0.3966,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.05626893788576126,
      "rewards/margins": 0.16169115900993347,
      "rewards/rejected": -0.10542222112417221,
      "step": 32
    },
    {
      "debug/policy_chosen_logits": -1.1181310415267944,
      "debug/policy_chosen_logps": -154.81201171875,
      "debug/policy_rejected_logits": -1.2310353517532349,
      "debug/policy_rejected_logps": -287.8173828125,
      "debug/reference_chosen_logps": -170.07876586914062,
      "debug/reference_rejected_logps": -274.1385498046875,
      "epoch": 0.515625,
      "grad_norm": 18.618810659036946,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1181310415267944,
      "logits/rejected": -1.2310353517532349,
      "logps/chosen": -154.81201171875,
      "logps/rejected": -287.8173828125,
      "loss": 0.3581,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1526675671339035,
      "rewards/margins": 0.2894558906555176,
      "rewards/rejected": -0.13678830862045288,
      "step": 33
    },
    {
      "debug/policy_chosen_logits": -1.0529826879501343,
      "debug/policy_chosen_logps": -128.18128967285156,
      "debug/policy_rejected_logits": -1.2277421951293945,
      "debug/policy_rejected_logps": -326.91705322265625,
      "debug/reference_chosen_logps": -147.74295043945312,
      "debug/reference_rejected_logps": -300.6445617675781,
      "epoch": 0.53125,
      "grad_norm": 19.76877319971208,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0529826879501343,
      "logits/rejected": -1.2277421951293945,
      "logps/chosen": -128.18128967285156,
      "logps/rejected": -326.91705322265625,
      "loss": 0.3702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19561666250228882,
      "rewards/margins": 0.4583418369293213,
      "rewards/rejected": -0.26272517442703247,
      "step": 34
    },
    {
      "debug/policy_chosen_logits": -1.0484968423843384,
      "debug/policy_chosen_logps": -177.14181518554688,
      "debug/policy_rejected_logits": -1.0831434726715088,
      "debug/policy_rejected_logps": -277.63067626953125,
      "debug/reference_chosen_logps": -184.79954528808594,
      "debug/reference_rejected_logps": -262.337646484375,
      "epoch": 0.546875,
      "grad_norm": 13.405977545151604,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0484968423843384,
      "logits/rejected": -1.0831434726715088,
      "logps/chosen": -177.14181518554688,
      "logps/rejected": -277.63067626953125,
      "loss": 0.3774,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.07657738029956818,
      "rewards/margins": 0.22950761020183563,
      "rewards/rejected": -0.15293022990226746,
      "step": 35
    },
    {
      "debug/policy_chosen_logits": -1.070804476737976,
      "debug/policy_chosen_logps": -119.32257080078125,
      "debug/policy_rejected_logits": -1.1960089206695557,
      "debug/policy_rejected_logps": -257.6097412109375,
      "debug/reference_chosen_logps": -134.1144561767578,
      "debug/reference_rejected_logps": -249.19239807128906,
      "epoch": 0.5625,
      "grad_norm": 54.78669264655883,
      "learning_rate": 1e-06,
      "logits/chosen": -1.070804476737976,
      "logits/rejected": -1.1960089206695557,
      "logps/chosen": -119.32257080078125,
      "logps/rejected": -257.6097412109375,
      "loss": 0.4202,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1479189097881317,
      "rewards/margins": 0.23209232091903687,
      "rewards/rejected": -0.08417341113090515,
      "step": 36
    },
    {
      "debug/policy_chosen_logits": -1.0936942100524902,
      "debug/policy_chosen_logps": -198.59994506835938,
      "debug/policy_rejected_logits": -1.1287853717803955,
      "debug/policy_rejected_logps": -267.81048583984375,
      "debug/reference_chosen_logps": -206.68980407714844,
      "debug/reference_rejected_logps": -260.12896728515625,
      "epoch": 0.578125,
      "grad_norm": 18.618309162410206,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0936942100524902,
      "logits/rejected": -1.1287853717803955,
      "logps/chosen": -198.59994506835938,
      "logps/rejected": -267.81048583984375,
      "loss": 0.3922,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08089858293533325,
      "rewards/margins": 0.15771383047103882,
      "rewards/rejected": -0.07681524008512497,
      "step": 37
    },
    {
      "debug/policy_chosen_logits": -1.0987818241119385,
      "debug/policy_chosen_logps": -156.1143798828125,
      "debug/policy_rejected_logits": -1.016094446182251,
      "debug/policy_rejected_logps": -280.1226806640625,
      "debug/reference_chosen_logps": -174.13986206054688,
      "debug/reference_rejected_logps": -272.59063720703125,
      "epoch": 0.59375,
      "grad_norm": 48.92722394829403,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0987818241119385,
      "logits/rejected": -1.016094446182251,
      "logps/chosen": -156.1143798828125,
      "logps/rejected": -280.1226806640625,
      "loss": 0.4235,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18025492131710052,
      "rewards/margins": 0.2555754780769348,
      "rewards/rejected": -0.07532056421041489,
      "step": 38
    },
    {
      "debug/policy_chosen_logits": -1.1431177854537964,
      "debug/policy_chosen_logps": -121.45298767089844,
      "debug/policy_rejected_logits": -1.2573899030685425,
      "debug/policy_rejected_logps": -243.77618408203125,
      "debug/reference_chosen_logps": -132.9182891845703,
      "debug/reference_rejected_logps": -236.6573486328125,
      "epoch": 0.609375,
      "grad_norm": 28.262320173832173,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1431177854537964,
      "logits/rejected": -1.2573899030685425,
      "logps/chosen": -121.45298767089844,
      "logps/rejected": -243.77618408203125,
      "loss": 0.3976,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11465291678905487,
      "rewards/margins": 0.18584111332893372,
      "rewards/rejected": -0.07118818163871765,
      "step": 39
    },
    {
      "debug/policy_chosen_logits": -1.1291528940200806,
      "debug/policy_chosen_logps": -124.88560485839844,
      "debug/policy_rejected_logits": -1.1997623443603516,
      "debug/policy_rejected_logps": -341.6507568359375,
      "debug/reference_chosen_logps": -145.1587677001953,
      "debug/reference_rejected_logps": -316.4557189941406,
      "epoch": 0.625,
      "grad_norm": 17.205504877297493,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1291528940200806,
      "logits/rejected": -1.1997623443603516,
      "logps/chosen": -124.88560485839844,
      "logps/rejected": -341.6507568359375,
      "loss": 0.3983,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.202731654047966,
      "rewards/margins": 0.4546818137168884,
      "rewards/rejected": -0.25195014476776123,
      "step": 40
    },
    {
      "debug/policy_chosen_logits": -1.119407057762146,
      "debug/policy_chosen_logps": -155.58392333984375,
      "debug/policy_rejected_logits": -1.165313720703125,
      "debug/policy_rejected_logps": -216.57156372070312,
      "debug/reference_chosen_logps": -161.89459228515625,
      "debug/reference_rejected_logps": -214.2755126953125,
      "epoch": 0.640625,
      "grad_norm": 20.732094832807366,
      "learning_rate": 1e-06,
      "logits/chosen": -1.119407057762146,
      "logits/rejected": -1.165313720703125,
      "logps/chosen": -155.58392333984375,
      "logps/rejected": -216.57156372070312,
      "loss": 0.3763,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06310684233903885,
      "rewards/margins": 0.0860673040151596,
      "rewards/rejected": -0.02296045981347561,
      "step": 41
    },
    {
      "debug/policy_chosen_logits": -1.2078365087509155,
      "debug/policy_chosen_logps": -137.1336212158203,
      "debug/policy_rejected_logits": -1.2154945135116577,
      "debug/policy_rejected_logps": -227.4922637939453,
      "debug/reference_chosen_logps": -139.9180145263672,
      "debug/reference_rejected_logps": -215.813232421875,
      "epoch": 0.65625,
      "grad_norm": 34.027873181354636,
      "learning_rate": 1e-06,
      "logits/chosen": -1.2078365087509155,
      "logits/rejected": -1.2154945135116577,
      "logps/chosen": -137.1336212158203,
      "logps/rejected": -227.4922637939453,
      "loss": 0.4182,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.02784401923418045,
      "rewards/margins": 0.14463430643081665,
      "rewards/rejected": -0.1167902946472168,
      "step": 42
    },
    {
      "debug/policy_chosen_logits": -1.0180912017822266,
      "debug/policy_chosen_logps": -173.8270263671875,
      "debug/policy_rejected_logits": -1.1830826997756958,
      "debug/policy_rejected_logps": -286.73638916015625,
      "debug/reference_chosen_logps": -174.58895874023438,
      "debug/reference_rejected_logps": -263.51458740234375,
      "epoch": 0.671875,
      "grad_norm": 26.885686366047068,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0180912017822266,
      "logits/rejected": -1.1830826997756958,
      "logps/chosen": -173.8270263671875,
      "logps/rejected": -286.73638916015625,
      "loss": 0.3939,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.007619347423315048,
      "rewards/margins": 0.2398374080657959,
      "rewards/rejected": -0.23221805691719055,
      "step": 43
    },
    {
      "debug/policy_chosen_logits": -0.8629423379898071,
      "debug/policy_chosen_logps": -186.4468994140625,
      "debug/policy_rejected_logits": -1.196955680847168,
      "debug/policy_rejected_logps": -291.8290710449219,
      "debug/reference_chosen_logps": -192.09939575195312,
      "debug/reference_rejected_logps": -283.04547119140625,
      "epoch": 0.6875,
      "grad_norm": 15.341359477798175,
      "learning_rate": 1e-06,
      "logits/chosen": -0.8629423379898071,
      "logits/rejected": -1.196955680847168,
      "logps/chosen": -186.4468994140625,
      "logps/rejected": -291.8290710449219,
      "loss": 0.3941,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05652495473623276,
      "rewards/margins": 0.14436087012290955,
      "rewards/rejected": -0.08783592283725739,
      "step": 44
    },
    {
      "debug/policy_chosen_logits": -1.1474281549453735,
      "debug/policy_chosen_logps": -185.1705322265625,
      "debug/policy_rejected_logits": -1.2113550901412964,
      "debug/policy_rejected_logps": -299.13165283203125,
      "debug/reference_chosen_logps": -184.02684020996094,
      "debug/reference_rejected_logps": -283.3847961425781,
      "epoch": 0.703125,
      "grad_norm": 27.424785120293386,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1474281549453735,
      "logits/rejected": -1.2113550901412964,
      "logps/chosen": -185.1705322265625,
      "logps/rejected": -299.13165283203125,
      "loss": 0.4015,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.011436812579631805,
      "rewards/margins": 0.1460317075252533,
      "rewards/rejected": -0.1574685126543045,
      "step": 45
    },
    {
      "debug/policy_chosen_logits": -1.0573773384094238,
      "debug/policy_chosen_logps": -127.71075439453125,
      "debug/policy_rejected_logits": -1.0924162864685059,
      "debug/policy_rejected_logps": -323.93768310546875,
      "debug/reference_chosen_logps": -139.21630859375,
      "debug/reference_rejected_logps": -311.1994323730469,
      "epoch": 0.71875,
      "grad_norm": 17.144934905131425,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0573773384094238,
      "logits/rejected": -1.0924162864685059,
      "logps/chosen": -127.71075439453125,
      "logps/rejected": -323.93768310546875,
      "loss": 0.3624,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11505550146102905,
      "rewards/margins": 0.24243810772895813,
      "rewards/rejected": -0.12738259136676788,
      "step": 46
    },
    {
      "debug/policy_chosen_logits": -1.0909616947174072,
      "debug/policy_chosen_logps": -137.27731323242188,
      "debug/policy_rejected_logits": -1.2138352394104004,
      "debug/policy_rejected_logps": -241.8701171875,
      "debug/reference_chosen_logps": -147.23553466796875,
      "debug/reference_rejected_logps": -222.49639892578125,
      "epoch": 0.734375,
      "grad_norm": 12.93169650628382,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0909616947174072,
      "logits/rejected": -1.2138352394104004,
      "logps/chosen": -137.27731323242188,
      "logps/rejected": -241.8701171875,
      "loss": 0.3217,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0995820164680481,
      "rewards/margins": 0.2933192849159241,
      "rewards/rejected": -0.19373726844787598,
      "step": 47
    },
    {
      "debug/policy_chosen_logits": -1.096240520477295,
      "debug/policy_chosen_logps": -232.75778198242188,
      "debug/policy_rejected_logits": -1.1766290664672852,
      "debug/policy_rejected_logps": -306.53369140625,
      "debug/reference_chosen_logps": -230.5318145751953,
      "debug/reference_rejected_logps": -294.82598876953125,
      "epoch": 0.75,
      "grad_norm": 26.099751982850893,
      "learning_rate": 1e-06,
      "logits/chosen": -1.096240520477295,
      "logits/rejected": -1.1766290664672852,
      "logps/chosen": -232.75778198242188,
      "logps/rejected": -306.53369140625,
      "loss": 0.4361,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.022259674966335297,
      "rewards/margins": 0.09481699019670486,
      "rewards/rejected": -0.11707665771245956,
      "step": 48
    },
    {
      "debug/policy_chosen_logits": -1.1644705533981323,
      "debug/policy_chosen_logps": -166.67062377929688,
      "debug/policy_rejected_logits": -1.293932557106018,
      "debug/policy_rejected_logps": -293.45050048828125,
      "debug/reference_chosen_logps": -175.53598022460938,
      "debug/reference_rejected_logps": -276.24322509765625,
      "epoch": 0.765625,
      "grad_norm": 18.426480334845714,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1644705533981323,
      "logits/rejected": -1.293932557106018,
      "logps/chosen": -166.67062377929688,
      "logps/rejected": -293.45050048828125,
      "loss": 0.4144,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08865345269441605,
      "rewards/margins": 0.2607261538505554,
      "rewards/rejected": -0.17207267880439758,
      "step": 49
    },
    {
      "debug/policy_chosen_logits": -1.2152189016342163,
      "debug/policy_chosen_logps": -170.15440368652344,
      "debug/policy_rejected_logits": -1.2675527334213257,
      "debug/policy_rejected_logps": -284.37353515625,
      "debug/reference_chosen_logps": -173.90533447265625,
      "debug/reference_rejected_logps": -265.96417236328125,
      "epoch": 0.78125,
      "grad_norm": 19.567832925259168,
      "learning_rate": 1e-06,
      "logits/chosen": -1.2152189016342163,
      "logits/rejected": -1.2675527334213257,
      "logps/chosen": -170.15440368652344,
      "logps/rejected": -284.37353515625,
      "loss": 0.3895,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03750941902399063,
      "rewards/margins": 0.22160324454307556,
      "rewards/rejected": -0.18409383296966553,
      "step": 50
    },
    {
      "debug/policy_chosen_logits": -1.1334317922592163,
      "debug/policy_chosen_logps": -127.97447204589844,
      "debug/policy_rejected_logits": -1.07590651512146,
      "debug/policy_rejected_logps": -220.5333251953125,
      "debug/reference_chosen_logps": -136.77487182617188,
      "debug/reference_rejected_logps": -219.20693969726562,
      "epoch": 0.796875,
      "grad_norm": 37.00007516828202,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1334317922592163,
      "logits/rejected": -1.07590651512146,
      "logps/chosen": -127.97447204589844,
      "logps/rejected": -220.5333251953125,
      "loss": 0.3521,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0880039781332016,
      "rewards/margins": 0.10126776248216629,
      "rewards/rejected": -0.013263778761029243,
      "step": 51
    },
    {
      "debug/policy_chosen_logits": -1.207089900970459,
      "debug/policy_chosen_logps": -149.59579467773438,
      "debug/policy_rejected_logits": -1.3598229885101318,
      "debug/policy_rejected_logps": -312.65423583984375,
      "debug/reference_chosen_logps": -160.83349609375,
      "debug/reference_rejected_logps": -290.1050109863281,
      "epoch": 0.8125,
      "grad_norm": 34.43193601355931,
      "learning_rate": 1e-06,
      "logits/chosen": -1.207089900970459,
      "logits/rejected": -1.3598229885101318,
      "logps/chosen": -149.59579467773438,
      "logps/rejected": -312.65423583984375,
      "loss": 0.3701,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11237694323062897,
      "rewards/margins": 0.3378693461418152,
      "rewards/rejected": -0.22549240291118622,
      "step": 52
    },
    {
      "debug/policy_chosen_logits": -1.0188125371932983,
      "debug/policy_chosen_logps": -185.8585205078125,
      "debug/policy_rejected_logits": -1.0791672468185425,
      "debug/policy_rejected_logps": -251.5456085205078,
      "debug/reference_chosen_logps": -191.01089477539062,
      "debug/reference_rejected_logps": -245.11524963378906,
      "epoch": 0.828125,
      "grad_norm": 14.59754124103045,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0188125371932983,
      "logits/rejected": -1.0791672468185425,
      "logps/chosen": -185.8585205078125,
      "logps/rejected": -251.5456085205078,
      "loss": 0.3737,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.051523588597774506,
      "rewards/margins": 0.1158272996544838,
      "rewards/rejected": -0.0643036961555481,
      "step": 53
    },
    {
      "debug/policy_chosen_logits": -1.0893926620483398,
      "debug/policy_chosen_logps": -149.98660278320312,
      "debug/policy_rejected_logits": -1.0650213956832886,
      "debug/policy_rejected_logps": -274.9080810546875,
      "debug/reference_chosen_logps": -158.46145629882812,
      "debug/reference_rejected_logps": -258.4507141113281,
      "epoch": 0.84375,
      "grad_norm": 14.810580428901549,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0893926620483398,
      "logits/rejected": -1.0650213956832886,
      "logps/chosen": -149.98660278320312,
      "logps/rejected": -274.9080810546875,
      "loss": 0.3213,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0847485214471817,
      "rewards/margins": 0.24932223558425903,
      "rewards/rejected": -0.16457369923591614,
      "step": 54
    },
    {
      "debug/policy_chosen_logits": -1.2362074851989746,
      "debug/policy_chosen_logps": -102.0992202758789,
      "debug/policy_rejected_logits": -1.3010079860687256,
      "debug/policy_rejected_logps": -289.4234313964844,
      "debug/reference_chosen_logps": -120.96076965332031,
      "debug/reference_rejected_logps": -275.486083984375,
      "epoch": 0.859375,
      "grad_norm": 13.99372317117744,
      "learning_rate": 1e-06,
      "logits/chosen": -1.2362074851989746,
      "logits/rejected": -1.3010079860687256,
      "logps/chosen": -102.0992202758789,
      "logps/rejected": -289.4234313964844,
      "loss": 0.4006,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18861553072929382,
      "rewards/margins": 0.32798925042152405,
      "rewards/rejected": -0.13937373459339142,
      "step": 55
    },
    {
      "debug/policy_chosen_logits": -1.0959794521331787,
      "debug/policy_chosen_logps": -176.76089477539062,
      "debug/policy_rejected_logits": -1.291311264038086,
      "debug/policy_rejected_logps": -313.87506103515625,
      "debug/reference_chosen_logps": -185.58998107910156,
      "debug/reference_rejected_logps": -290.28045654296875,
      "epoch": 0.875,
      "grad_norm": 37.669129247782706,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0959794521331787,
      "logits/rejected": -1.291311264038086,
      "logps/chosen": -176.76089477539062,
      "logps/rejected": -313.87506103515625,
      "loss": 0.3289,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.08829064667224884,
      "rewards/margins": 0.3242364823818207,
      "rewards/rejected": -0.23594582080841064,
      "step": 56
    },
    {
      "debug/policy_chosen_logits": -1.0723934173583984,
      "debug/policy_chosen_logps": -127.6189193725586,
      "debug/policy_rejected_logits": -1.1941779851913452,
      "debug/policy_rejected_logps": -263.9356689453125,
      "debug/reference_chosen_logps": -139.6109161376953,
      "debug/reference_rejected_logps": -251.62448120117188,
      "epoch": 0.890625,
      "grad_norm": 15.916622092420505,
      "learning_rate": 1e-06,
      "logits/chosen": -1.0723934173583984,
      "logits/rejected": -1.1941779851913452,
      "logps/chosen": -127.6189193725586,
      "logps/rejected": -263.9356689453125,
      "loss": 0.3641,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11991991102695465,
      "rewards/margins": 0.24303147196769714,
      "rewards/rejected": -0.12311156839132309,
      "step": 57
    },
    {
      "debug/policy_chosen_logits": -1.083500862121582,
      "debug/policy_chosen_logps": -212.94515991210938,
      "debug/policy_rejected_logits": -1.196679711341858,
      "debug/policy_rejected_logps": -263.7575378417969,
      "debug/reference_chosen_logps": -221.95928955078125,
      "debug/reference_rejected_logps": -267.07586669921875,
      "epoch": 0.90625,
      "grad_norm": 26.520012974267605,
      "learning_rate": 1e-06,
      "logits/chosen": -1.083500862121582,
      "logits/rejected": -1.196679711341858,
      "logps/chosen": -212.94515991210938,
      "logps/rejected": -263.7575378417969,
      "loss": 0.4082,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.09014149010181427,
      "rewards/margins": 0.05695834010839462,
      "rewards/rejected": 0.03318314626812935,
      "step": 58
    },
    {
      "debug/policy_chosen_logits": -1.2750979661941528,
      "debug/policy_chosen_logps": -120.48554229736328,
      "debug/policy_rejected_logits": -1.2684656381607056,
      "debug/policy_rejected_logps": -331.54986572265625,
      "debug/reference_chosen_logps": -130.17742919921875,
      "debug/reference_rejected_logps": -307.9356689453125,
      "epoch": 0.921875,
      "grad_norm": 19.676716039926774,
      "learning_rate": 1e-06,
      "logits/chosen": -1.2750979661941528,
      "logits/rejected": -1.2684656381607056,
      "logps/chosen": -120.48554229736328,
      "logps/rejected": -331.54986572265625,
      "loss": 0.3486,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09691886603832245,
      "rewards/margins": 0.3330605924129486,
      "rewards/rejected": -0.23614171147346497,
      "step": 59
    },
    {
      "debug/policy_chosen_logits": -1.1700026988983154,
      "debug/policy_chosen_logps": -164.0662078857422,
      "debug/policy_rejected_logits": -1.0647200345993042,
      "debug/policy_rejected_logps": -289.3599548339844,
      "debug/reference_chosen_logps": -172.45896911621094,
      "debug/reference_rejected_logps": -276.63592529296875,
      "epoch": 0.9375,
      "grad_norm": 16.5920657057711,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1700026988983154,
      "logits/rejected": -1.0647200345993042,
      "logps/chosen": -164.0662078857422,
      "logps/rejected": -289.3599548339844,
      "loss": 0.3551,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08392763137817383,
      "rewards/margins": 0.21116778254508972,
      "rewards/rejected": -0.1272401511669159,
      "step": 60
    },
    {
      "debug/policy_chosen_logits": -1.2429842948913574,
      "debug/policy_chosen_logps": -164.30996704101562,
      "debug/policy_rejected_logits": -1.2771668434143066,
      "debug/policy_rejected_logps": -291.66436767578125,
      "debug/reference_chosen_logps": -178.3618927001953,
      "debug/reference_rejected_logps": -263.1362609863281,
      "epoch": 0.953125,
      "grad_norm": 17.917957649513887,
      "learning_rate": 1e-06,
      "logits/chosen": -1.2429842948913574,
      "logits/rejected": -1.2771668434143066,
      "logps/chosen": -164.30996704101562,
      "logps/rejected": -291.66436767578125,
      "loss": 0.333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14051929116249084,
      "rewards/margins": 0.42580026388168335,
      "rewards/rejected": -0.2852809429168701,
      "step": 61
    },
    {
      "debug/policy_chosen_logits": -1.1629077196121216,
      "debug/policy_chosen_logps": -171.4347381591797,
      "debug/policy_rejected_logits": -1.2383259534835815,
      "debug/policy_rejected_logps": -257.24322509765625,
      "debug/reference_chosen_logps": -176.8075408935547,
      "debug/reference_rejected_logps": -236.5648193359375,
      "epoch": 0.96875,
      "grad_norm": 27.22416658714319,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1629077196121216,
      "logits/rejected": -1.2383259534835815,
      "logps/chosen": -171.4347381591797,
      "logps/rejected": -257.24322509765625,
      "loss": 0.3685,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.053728047758340836,
      "rewards/margins": 0.2605122923851013,
      "rewards/rejected": -0.20678424835205078,
      "step": 62
    },
    {
      "debug/policy_chosen_logits": -1.167179822921753,
      "debug/policy_chosen_logps": -241.232666015625,
      "debug/policy_rejected_logits": -1.1904563903808594,
      "debug/policy_rejected_logps": -349.2745361328125,
      "debug/reference_chosen_logps": -237.09837341308594,
      "debug/reference_rejected_logps": -312.7959289550781,
      "epoch": 0.984375,
      "grad_norm": 35.440096057306455,
      "learning_rate": 1e-06,
      "logits/chosen": -1.167179822921753,
      "logits/rejected": -1.1904563903808594,
      "logps/chosen": -241.232666015625,
      "logps/rejected": -349.2745361328125,
      "loss": 0.3671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04134296253323555,
      "rewards/margins": 0.32344281673431396,
      "rewards/rejected": -0.3647857904434204,
      "step": 63
    },
    {
      "debug/policy_chosen_logits": -1.1739040613174438,
      "debug/policy_chosen_logps": -139.26182556152344,
      "debug/policy_rejected_logits": -1.2884361743927002,
      "debug/policy_rejected_logps": -280.71124267578125,
      "debug/reference_chosen_logps": -148.495361328125,
      "debug/reference_rejected_logps": -259.9752197265625,
      "epoch": 1.0,
      "grad_norm": 43.313159052812985,
      "learning_rate": 1e-06,
      "logits/chosen": -1.1739040613174438,
      "logits/rejected": -1.2884361743927002,
      "logps/chosen": -139.26182556152344,
      "logps/rejected": -280.71124267578125,
      "loss": 0.3453,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.09233523905277252,
      "rewards/margins": 0.2996952533721924,
      "rewards/rejected": -0.20735999941825867,
      "step": 64
    },
    {
      "epoch": 1.0,
      "step": 64,
      "total_flos": 0.0,
      "train_loss": 0.40815131505951285,
      "train_runtime": 194.2921,
      "train_samples_per_second": 20.979,
      "train_steps_per_second": 0.329
    }
  ],
  "logging_steps": 1,
  "max_steps": 64,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}