diff --git "a/trainer_state.json" "b/trainer_state.json"
new file mode 100644--- /dev/null
+++ "b/trainer_state.json"
@@ -0,0 +1,7158 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 2.988679245283019,
+  "eval_steps": 500,
+  "global_step": 396,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.01,
+      "grad_norm": 6.326096763058934,
+      "learning_rate": 1.25e-08,
+      "logps/chosen": -36.02279281616211,
+      "logps/rejected": -41.85474395751953,
+      "loss": 0.6931,
+      "losses/dpo": 0.6931471824645996,
+      "losses/sft": 1.3949329853057861,
+      "losses/total": 0.6931471824645996,
+      "ref_logps/chosen": -36.02279281616211,
+      "ref_logps/rejected": -41.85474395751953,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 1
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 5.576784855809719,
+      "learning_rate": 2.5e-08,
+      "logps/chosen": -33.77919387817383,
+      "logps/rejected": -41.04405975341797,
+      "loss": 0.6931,
+      "losses/dpo": 0.6931471824645996,
+      "losses/sft": 1.3951497077941895,
+      "losses/total": 0.6931471824645996,
+      "ref_logps/chosen": -33.77919387817383,
+      "ref_logps/rejected": -41.04405975341797,
+      "rewards/accuracies": 0.0,
+      "rewards/chosen": 0.0,
+      "rewards/margins": 0.0,
+      "rewards/rejected": 0.0,
+      "step": 2
+    },
+    {
+      "epoch": 0.02,
+      "grad_norm": 6.263050301271669,
+      "learning_rate": 3.75e-08,
+      "logps/chosen": -38.8697509765625,
+      "logps/rejected": -48.85557556152344,
+      "loss": 0.6931,
+      "losses/dpo": 0.6860073804855347,
+      "losses/sft": 1.6376307010650635,
+      "losses/total": 0.6860073804855347,
+      "ref_logps/chosen": -38.87074279785156,
+      "ref_logps/rejected": -48.853511810302734,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 9.899254655465484e-05,
+      "rewards/margins": 0.0003055855631828308,
+      "rewards/rejected": -0.0002065933949779719,
+      "step": 3
+    },
+    {
+      "epoch": 0.03,
+      "grad_norm": 5.738951829533344,
+      "learning_rate": 5e-08,
+      "logps/chosen": -36.64889144897461,
+      "logps/rejected": -42.698097229003906,
+      "loss": 0.6924,
+      "losses/dpo": 0.6935421228408813,
+      "losses/sft": 1.4897900819778442,
+      "losses/total": 0.6935421228408813,
+      "ref_logps/chosen": -36.668033599853516,
+      "ref_logps/rejected": -42.70002746582031,
+      "rewards/accuracies": 0.484375,
+      "rewards/chosen": 0.0019142806995660067,
+      "rewards/margins": 0.001721527660265565,
+      "rewards/rejected": 0.00019275324302725494,
+      "step": 4
+    },
+    {
+      "epoch": 0.04,
+      "grad_norm": 6.637504299884417,
+      "learning_rate": 6.25e-08,
+      "logps/chosen": -41.41233825683594,
+      "logps/rejected": -47.04777145385742,
+      "loss": 0.6939,
+      "losses/dpo": 0.6956198215484619,
+      "losses/sft": 1.1974728107452393,
+      "losses/total": 0.6956198215484619,
+      "ref_logps/chosen": -41.40231704711914,
+      "ref_logps/rejected": -47.051856994628906,
+      "rewards/accuracies": 0.46875,
+      "rewards/chosen": -0.0010022701462730765,
+      "rewards/margins": -0.0014111174969002604,
+      "rewards/rejected": 0.00040884732152335346,
+      "step": 5
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 5.760428498194468,
+      "learning_rate": 7.5e-08,
+      "logps/chosen": -34.51856994628906,
+      "logps/rejected": -41.675804138183594,
+      "loss": 0.6946,
+      "losses/dpo": 0.6942625641822815,
+      "losses/sft": 1.3869932889938354,
+      "losses/total": 0.6942625641822815,
+      "ref_logps/chosen": -34.49778747558594,
+      "ref_logps/rejected": -41.68275451660156,
+      "rewards/accuracies": 0.421875,
+      "rewards/chosen": -0.0020779455080628395,
+      "rewards/margins": -0.002773313783109188,
+      "rewards/rejected": 0.0006953685078769922,
+      "step": 6
+    },
+    {
+      "epoch": 0.05,
+      "grad_norm": 5.773272972704967,
+      "learning_rate": 8.75e-08,
+      "logps/chosen": -36.6628303527832,
+      "logps/rejected": -42.856834411621094,
+      "loss": 0.6927,
+      "losses/dpo": 0.6944370269775391,
+      "losses/sft": 1.2695866823196411,
+      "losses/total": 0.6944370269775391,
+      "ref_logps/chosen": -36.669891357421875,
+      "ref_logps/rejected": -42.8528938293457,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": 0.0007062811637297273,
+      "rewards/margins": 0.0011000875383615494,
+      "rewards/rejected": -0.00039380654925480485,
+      "step": 7
+    },
+    {
+      "epoch": 0.06,
+      "grad_norm": 6.837371123256996,
+      "learning_rate": 1e-07,
+      "logps/chosen": -41.66258239746094,
+      "logps/rejected": -43.34931182861328,
+      "loss": 0.6943,
+      "losses/dpo": 0.6919102668762207,
+      "losses/sft": 1.317617654800415,
+      "losses/total": 0.6919102668762207,
+      "ref_logps/chosen": -41.65662384033203,
+      "ref_logps/rejected": -43.36621856689453,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.0005958047113381326,
+      "rewards/margins": -0.0022863391786813736,
+      "rewards/rejected": 0.0016905345255509019,
+      "step": 8
+    },
+    {
+      "epoch": 0.07,
+      "grad_norm": 6.116952924505858,
+      "learning_rate": 1.125e-07,
+      "logps/chosen": -37.05712890625,
+      "logps/rejected": -46.517696380615234,
+      "loss": 0.6917,
+      "losses/dpo": 0.692311704158783,
+      "losses/sft": 1.112796664237976,
+      "losses/total": 0.692311704158783,
+      "ref_logps/chosen": -37.07765197753906,
+      "ref_logps/rejected": -46.507904052734375,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": 0.0020526223815977573,
+      "rewards/margins": 0.003031900618225336,
+      "rewards/rejected": -0.0009792782366275787,
+      "step": 9
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 5.761856386512759,
+      "learning_rate": 1.25e-07,
+      "logps/chosen": -33.799774169921875,
+      "logps/rejected": -41.23558044433594,
+      "loss": 0.6924,
+      "losses/dpo": 0.6941465139389038,
+      "losses/sft": 1.1185486316680908,
+      "losses/total": 0.6941465139389038,
+      "ref_logps/chosen": -33.81248474121094,
+      "ref_logps/rejected": -41.23206329345703,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": 0.0012711097951978445,
+      "rewards/margins": 0.0016234376234933734,
+      "rewards/rejected": -0.0003523279447108507,
+      "step": 10
+    },
+    {
+      "epoch": 0.08,
+      "grad_norm": 5.743856364003174,
+      "learning_rate": 1.375e-07,
+      "logps/chosen": -36.227317810058594,
+      "logps/rejected": -40.51737976074219,
+      "loss": 0.6927,
+      "losses/dpo": 0.6916883587837219,
+      "losses/sft": 1.4357692003250122,
+      "losses/total": 0.6916883587837219,
+      "ref_logps/chosen": -36.23785400390625,
+      "ref_logps/rejected": -40.51884078979492,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": 0.0010535644832998514,
+      "rewards/margins": 0.0009071138338185847,
+      "rewards/rejected": 0.0001464505330659449,
+      "step": 11
+    },
+    {
+      "epoch": 0.09,
+      "grad_norm": 5.9204140360711355,
+      "learning_rate": 1.5e-07,
+      "logps/chosen": -38.817134857177734,
+      "logps/rejected": -42.217681884765625,
+      "loss": 0.6925,
+      "losses/dpo": 0.6985405683517456,
+      "losses/sft": 1.4544084072113037,
+      "losses/total": 0.6985405683517456,
+      "ref_logps/chosen": -38.83327102661133,
+      "ref_logps/rejected": -42.220176696777344,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": 0.0016135365003719926,
+      "rewards/margins": 0.0013644276186823845,
+      "rewards/rejected": 0.0002491088816896081,
+      "step": 12
+    },
+    {
+      "epoch": 0.1,
+      "grad_norm": 6.190779724671626,
+      "learning_rate": 1.625e-07,
+      "logps/chosen": -37.33137512207031,
+      "logps/rejected": -46.71794128417969,
+      "loss": 0.6901,
+      "losses/dpo": 0.6943342685699463,
+      "losses/sft": 1.3390721082687378,
+      "losses/total": 0.6943342685699463,
+      "ref_logps/chosen": -37.34603500366211,
+      "ref_logps/rejected": -46.670997619628906,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": 0.0014661503955721855,
+      "rewards/margins": 0.006160825490951538,
+      "rewards/rejected": -0.004694675095379353,
+      "step": 13
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 5.535961166630566,
+      "learning_rate": 1.75e-07,
+      "logps/chosen": -34.35616683959961,
+      "logps/rejected": -40.568878173828125,
+      "loss": 0.6923,
+      "losses/dpo": 0.6914072036743164,
+      "losses/sft": 1.0790843963623047,
+      "losses/total": 0.6914072036743164,
+      "ref_logps/chosen": -34.35405731201172,
+      "ref_logps/rejected": -40.548362731933594,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": -0.00021077337441965938,
+      "rewards/margins": 0.001840681186877191,
+      "rewards/rejected": -0.002051454270258546,
+      "step": 14
+    },
+    {
+      "epoch": 0.11,
+      "grad_norm": 5.994324182587906,
+      "learning_rate": 1.875e-07,
+      "logps/chosen": -35.86518859863281,
+      "logps/rejected": -41.03656005859375,
+      "loss": 0.6932,
+      "losses/dpo": 0.690587043762207,
+      "losses/sft": 1.5208988189697266,
+      "losses/total": 0.690587043762207,
+      "ref_logps/chosen": -35.85986328125,
+      "ref_logps/rejected": -41.031028747558594,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.0005322899669408798,
+      "rewards/margins": 2.0701438188552856e-05,
+      "rewards/rejected": -0.00055299187079072,
+      "step": 15
+    },
+    {
+      "epoch": 0.12,
+      "grad_norm": 5.908538195984286,
+      "learning_rate": 2e-07,
+      "logps/chosen": -36.70806884765625,
+      "logps/rejected": -39.733882904052734,
+      "loss": 0.6926,
+      "losses/dpo": 0.6891317963600159,
+      "losses/sft": 1.1712957620620728,
+      "losses/total": 0.6891317963600159,
+      "ref_logps/chosen": -36.713016510009766,
+      "ref_logps/rejected": -39.726341247558594,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": 0.0004950040020048618,
+      "rewards/margins": 0.0012491128873080015,
+      "rewards/rejected": -0.0007541090017184615,
+      "step": 16
+    },
+    {
+      "epoch": 0.13,
+      "grad_norm": 6.17892190890487,
+      "learning_rate": 2.1249999999999998e-07,
+      "logps/chosen": -38.60041046142578,
+      "logps/rejected": -43.30579376220703,
+      "loss": 0.694,
+      "losses/dpo": 0.6916015148162842,
+      "losses/sft": 1.3250274658203125,
+      "losses/total": 0.6916015148162842,
+      "ref_logps/chosen": -38.579288482666016,
+      "ref_logps/rejected": -43.300140380859375,
+      "rewards/accuracies": 0.4609375,
+      "rewards/chosen": -0.0021122824400663376,
+      "rewards/margins": -0.001547331572510302,
+      "rewards/rejected": -0.0005649511003866792,
+      "step": 17
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 6.020104058580606,
+      "learning_rate": 2.25e-07,
+      "logps/chosen": -37.50771713256836,
+      "logps/rejected": -41.76362609863281,
+      "loss": 0.6919,
+      "losses/dpo": 0.6925865411758423,
+      "losses/sft": 1.3761729001998901,
+      "losses/total": 0.6925865411758423,
+      "ref_logps/chosen": -37.507423400878906,
+      "ref_logps/rejected": -41.736366271972656,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -2.9282993637025356e-05,
+      "rewards/margins": 0.0026969274040311575,
+      "rewards/rejected": -0.002726210281252861,
+      "step": 18
+    },
+    {
+      "epoch": 0.14,
+      "grad_norm": 5.765433605501229,
+      "learning_rate": 2.3749999999999998e-07,
+      "logps/chosen": -32.96650695800781,
+      "logps/rejected": -42.942771911621094,
+      "loss": 0.6917,
+      "losses/dpo": 0.6924209594726562,
+      "losses/sft": 1.1748046875,
+      "losses/total": 0.6924209594726562,
+      "ref_logps/chosen": -32.971160888671875,
+      "ref_logps/rejected": -42.91703796386719,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": 0.00046532286796718836,
+      "rewards/margins": 0.0030381008982658386,
+      "rewards/rejected": -0.002572778146713972,
+      "step": 19
+    },
+    {
+      "epoch": 0.15,
+      "grad_norm": 5.824641204085951,
+      "learning_rate": 2.5e-07,
+      "logps/chosen": -37.962188720703125,
+      "logps/rejected": -43.213279724121094,
+      "loss": 0.6932,
+      "losses/dpo": 0.694710373878479,
+      "losses/sft": 1.2030720710754395,
+      "losses/total": 0.694710373878479,
+      "ref_logps/chosen": -37.919189453125,
+      "ref_logps/rejected": -43.171043395996094,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": -0.0043003251776099205,
+      "rewards/margins": -7.65085278544575e-05,
+      "rewards/rejected": -0.004223817028105259,
+      "step": 20
+    },
+    {
+      "epoch": 0.16,
+      "grad_norm": 6.193915477797873,
+      "learning_rate": 2.625e-07,
+      "logps/chosen": -37.831146240234375,
+      "logps/rejected": -47.295005798339844,
+      "loss": 0.6954,
+      "losses/dpo": 0.6994068622589111,
+      "losses/sft": 1.2481111288070679,
+      "losses/total": 0.6994068622589111,
+      "ref_logps/chosen": -37.76181411743164,
+      "ref_logps/rejected": -47.269264221191406,
+      "rewards/accuracies": 0.4453125,
+      "rewards/chosen": -0.006933108903467655,
+      "rewards/margins": -0.004359052516520023,
+      "rewards/rejected": -0.002574056386947632,
+      "step": 21
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 5.703247411276419,
+      "learning_rate": 2.75e-07,
+      "logps/chosen": -34.446189880371094,
+      "logps/rejected": -42.82508850097656,
+      "loss": 0.6939,
+      "losses/dpo": 0.6885063648223877,
+      "losses/sft": 1.2574893236160278,
+      "losses/total": 0.6885063648223877,
+      "ref_logps/chosen": -34.39026641845703,
+      "ref_logps/rejected": -42.782203674316406,
+      "rewards/accuracies": 0.4921875,
+      "rewards/chosen": -0.005592696368694305,
+      "rewards/margins": -0.0013045003870502114,
+      "rewards/rejected": -0.004288196098059416,
+      "step": 22
+    },
+    {
+      "epoch": 0.17,
+      "grad_norm": 5.586186575504996,
+      "learning_rate": 2.8749999999999995e-07,
+      "logps/chosen": -35.78218078613281,
+      "logps/rejected": -46.140350341796875,
+      "loss": 0.6929,
+      "losses/dpo": 0.6936126351356506,
+      "losses/sft": 1.4059488773345947,
+      "losses/total": 0.6936126351356506,
+      "ref_logps/chosen": -35.713783264160156,
+      "ref_logps/rejected": -46.065738677978516,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.00683995708823204,
+      "rewards/margins": 0.0006212468724697828,
+      "rewards/rejected": -0.00746120372787118,
+      "step": 23
+    },
+    {
+      "epoch": 0.18,
+      "grad_norm": 6.2807240194752,
+      "learning_rate": 3e-07,
+      "logps/chosen": -37.896644592285156,
+      "logps/rejected": -43.448909759521484,
+      "loss": 0.6914,
+      "losses/dpo": 0.6850650310516357,
+      "losses/sft": 1.4576250314712524,
+      "losses/total": 0.6850650310516357,
+      "ref_logps/chosen": -37.83003616333008,
+      "ref_logps/rejected": -43.34458923339844,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": -0.00666093360632658,
+      "rewards/margins": 0.00377137353643775,
+      "rewards/rejected": -0.010432307608425617,
+      "step": 24
+    },
+    {
+      "epoch": 0.19,
+      "grad_norm": 5.6714251513252485,
+      "learning_rate": 3.1249999999999997e-07,
+      "logps/chosen": -36.5435791015625,
+      "logps/rejected": -41.46415710449219,
+      "loss": 0.6923,
+      "losses/dpo": 0.6902315020561218,
+      "losses/sft": 1.3371169567108154,
+      "losses/total": 0.6902315020561218,
+      "ref_logps/chosen": -36.474647521972656,
+      "ref_logps/rejected": -41.37662887573242,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": -0.006893564946949482,
+      "rewards/margins": 0.001859544194303453,
+      "rewards/rejected": -0.00875310879200697,
+      "step": 25
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 6.279222054280467,
+      "learning_rate": 3.25e-07,
+      "logps/chosen": -37.0484733581543,
+      "logps/rejected": -44.5318603515625,
+      "loss": 0.6919,
+      "losses/dpo": 0.6916804313659668,
+      "losses/sft": 1.2641081809997559,
+      "losses/total": 0.6916804313659668,
+      "ref_logps/chosen": -36.94280242919922,
+      "ref_logps/rejected": -44.40016174316406,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.010566946119070053,
+      "rewards/margins": 0.0026026167906820774,
+      "rewards/rejected": -0.013169562444090843,
+      "step": 26
+    },
+    {
+      "epoch": 0.2,
+      "grad_norm": 5.569722658657549,
+      "learning_rate": 3.375e-07,
+      "logps/chosen": -33.24622344970703,
+      "logps/rejected": -39.62266540527344,
+      "loss": 0.6918,
+      "losses/dpo": 0.6929441094398499,
+      "losses/sft": 1.0789260864257812,
+      "losses/total": 0.6929441094398499,
+      "ref_logps/chosen": -33.152587890625,
+      "ref_logps/rejected": -39.500396728515625,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.009363781660795212,
+      "rewards/margins": 0.002862950786948204,
+      "rewards/rejected": -0.012226731516420841,
+      "step": 27
+    },
+    {
+      "epoch": 0.21,
+      "grad_norm": 6.178096448266021,
+      "learning_rate": 3.5e-07,
+      "logps/chosen": -40.909210205078125,
+      "logps/rejected": -43.54678726196289,
+      "loss": 0.6932,
+      "losses/dpo": 0.6962201595306396,
+      "losses/sft": 1.4941421747207642,
+      "losses/total": 0.6962201595306396,
+      "ref_logps/chosen": -40.74734878540039,
+      "ref_logps/rejected": -43.38502883911133,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -0.016186244785785675,
+      "rewards/margins": -1.0382413165643811e-05,
+      "rewards/rejected": -0.01617586426436901,
+      "step": 28
+    },
+    {
+      "epoch": 0.22,
+      "grad_norm": 6.351428202937179,
+      "learning_rate": 3.6249999999999997e-07,
+      "logps/chosen": -38.18675231933594,
+      "logps/rejected": -45.641944885253906,
+      "loss": 0.6918,
+      "losses/dpo": 0.6936982870101929,
+      "losses/sft": 1.5615143775939941,
+      "losses/total": 0.6936982870101929,
+      "ref_logps/chosen": -38.07172775268555,
+      "ref_logps/rejected": -45.49729919433594,
+      "rewards/accuracies": 0.546875,
+      "rewards/chosen": -0.011502932757139206,
+      "rewards/margins": 0.002961072139441967,
+      "rewards/rejected": -0.014464004896581173,
+      "step": 29
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 6.197352455942284,
+      "learning_rate": 3.75e-07,
+      "logps/chosen": -38.10821533203125,
+      "logps/rejected": -44.8267707824707,
+      "loss": 0.6913,
+      "losses/dpo": 0.6886826753616333,
+      "losses/sft": 1.2551246881484985,
+      "losses/total": 0.6886826753616333,
+      "ref_logps/chosen": -37.95869827270508,
+      "ref_logps/rejected": -44.637386322021484,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": -0.014951780438423157,
+      "rewards/margins": 0.00398671068251133,
+      "rewards/rejected": -0.018938491120934486,
+      "step": 30
+    },
+    {
+      "epoch": 0.23,
+      "grad_norm": 6.109331160092928,
+      "learning_rate": 3.875e-07,
+      "logps/chosen": -37.73468017578125,
+      "logps/rejected": -45.03502655029297,
+      "loss": 0.6939,
+      "losses/dpo": 0.6942582130432129,
+      "losses/sft": 1.307703971862793,
+      "losses/total": 0.6942582130432129,
+      "ref_logps/chosen": -37.55635070800781,
+      "ref_logps/rejected": -44.870731353759766,
+      "rewards/accuracies": 0.4375,
+      "rewards/chosen": -0.01783285290002823,
+      "rewards/margins": -0.0014032268663868308,
+      "rewards/rejected": -0.01642962545156479,
+      "step": 31
+    },
+    {
+      "epoch": 0.24,
+      "grad_norm": 5.667604081077805,
+      "learning_rate": 4e-07,
+      "logps/chosen": -35.0442008972168,
+      "logps/rejected": -43.61913299560547,
+      "loss": 0.6936,
+      "losses/dpo": 0.7014378309249878,
+      "losses/sft": 1.3467621803283691,
+      "losses/total": 0.7014378309249878,
+      "ref_logps/chosen": -34.83974838256836,
+      "ref_logps/rejected": -43.42043685913086,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.02044512704014778,
+      "rewards/margins": -0.0005755843594670296,
+      "rewards/rejected": -0.019869543612003326,
+      "step": 32
+    },
+    {
+      "epoch": 0.25,
+      "grad_norm": 5.678502382891613,
+      "learning_rate": 4.1249999999999997e-07,
+      "logps/chosen": -32.68036651611328,
+      "logps/rejected": -37.5178337097168,
+      "loss": 0.6907,
+      "losses/dpo": 0.6907713413238525,
+      "losses/sft": 1.0900930166244507,
+      "losses/total": 0.6907713413238525,
+      "ref_logps/chosen": -32.45442199707031,
+      "ref_logps/rejected": -37.23982238769531,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.0225942712277174,
+      "rewards/margins": 0.005206821020692587,
+      "rewards/rejected": -0.027801092714071274,
+      "step": 33
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 5.921573916861138,
+      "learning_rate": 4.2499999999999995e-07,
+      "logps/chosen": -38.70556640625,
+      "logps/rejected": -44.084251403808594,
+      "loss": 0.6917,
+      "losses/dpo": 0.6954081058502197,
+      "losses/sft": 1.3240528106689453,
+      "losses/total": 0.6954081058502197,
+      "ref_logps/chosen": -38.448394775390625,
+      "ref_logps/rejected": -43.795318603515625,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": -0.025717251002788544,
+      "rewards/margins": 0.0031764586456120014,
+      "rewards/rejected": -0.02889370732009411,
+      "step": 34
+    },
+    {
+      "epoch": 0.26,
+      "grad_norm": 6.262461804418144,
+      "learning_rate": 4.375e-07,
+      "logps/chosen": -38.28644561767578,
+      "logps/rejected": -44.10706329345703,
+      "loss": 0.6906,
+      "losses/dpo": 0.6915950179100037,
+      "losses/sft": 1.5137853622436523,
+      "losses/total": 0.6915950179100037,
+      "ref_logps/chosen": -37.993186950683594,
+      "ref_logps/rejected": -43.75933837890625,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": -0.029325801879167557,
+      "rewards/margins": 0.005446841474622488,
+      "rewards/rejected": -0.03477264195680618,
+      "step": 35
+    },
+    {
+      "epoch": 0.27,
+      "grad_norm": 6.51719678701152,
+      "learning_rate": 4.5e-07,
+      "logps/chosen": -38.827396392822266,
+      "logps/rejected": -44.608299255371094,
+      "loss": 0.6898,
+      "losses/dpo": 0.6939886212348938,
+      "losses/sft": 1.1804178953170776,
+      "losses/total": 0.6939886212348938,
+      "ref_logps/chosen": -38.524925231933594,
+      "ref_logps/rejected": -44.23436737060547,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -0.03024711087346077,
+      "rewards/margins": 0.007146051619201899,
+      "rewards/rejected": -0.03739316016435623,
+      "step": 36
+    },
+    {
+      "epoch": 0.28,
+      "grad_norm": 6.054369450491099,
+      "learning_rate": 4.625e-07,
+      "logps/chosen": -37.75469970703125,
+      "logps/rejected": -40.86686706542969,
+      "loss": 0.69,
+      "losses/dpo": 0.6901004910469055,
+      "losses/sft": 1.2039740085601807,
+      "losses/total": 0.6901004910469055,
+      "ref_logps/chosen": -37.38399124145508,
+      "ref_logps/rejected": -40.42985534667969,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.03707098215818405,
+      "rewards/margins": 0.006630584131926298,
+      "rewards/rejected": -0.04370156675577164,
+      "step": 37
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 5.811128045517565,
+      "learning_rate": 4.7499999999999995e-07,
+      "logps/chosen": -37.347511291503906,
+      "logps/rejected": -42.597320556640625,
+      "loss": 0.6932,
+      "losses/dpo": 0.6947627663612366,
+      "losses/sft": 1.5172080993652344,
+      "losses/total": 0.6947627663612366,
+      "ref_logps/chosen": -36.92784881591797,
+      "ref_logps/rejected": -42.17408752441406,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": -0.041966233402490616,
+      "rewards/margins": 0.00035727641079574823,
+      "rewards/rejected": -0.042323507368564606,
+      "step": 38
+    },
+    {
+      "epoch": 0.29,
+      "grad_norm": 6.232125527781941,
+      "learning_rate": 4.875e-07,
+      "logps/chosen": -35.76224899291992,
+      "logps/rejected": -40.480010986328125,
+      "loss": 0.6877,
+      "losses/dpo": 0.6863731741905212,
+      "losses/sft": 1.403287410736084,
+      "losses/total": 0.6863731741905212,
+      "ref_logps/chosen": -35.296600341796875,
+      "ref_logps/rejected": -39.899620056152344,
+      "rewards/accuracies": 0.5390625,
+      "rewards/chosen": -0.04656480997800827,
+      "rewards/margins": 0.011474234983325005,
+      "rewards/rejected": -0.058039046823978424,
+      "step": 39
+    },
+    {
+      "epoch": 0.3,
+      "grad_norm": 5.919014140290479,
+      "learning_rate": 5e-07,
+      "logps/chosen": -33.405452728271484,
+      "logps/rejected": -40.23749542236328,
+      "loss": 0.6917,
+      "losses/dpo": 0.7027544975280762,
+      "losses/sft": 1.5135366916656494,
+      "losses/total": 0.7027544975280762,
+      "ref_logps/chosen": -32.92824935913086,
+      "ref_logps/rejected": -39.72548294067383,
+      "rewards/accuracies": 0.5703125,
+      "rewards/chosen": -0.04772059991955757,
+      "rewards/margins": 0.0034805855248123407,
+      "rewards/rejected": -0.05120118334889412,
+      "step": 40
+    },
+    {
+      "epoch": 0.31,
+      "grad_norm": 6.156564024356789,
+      "learning_rate": 4.985955056179775e-07,
+      "logps/chosen": -33.48844528198242,
+      "logps/rejected": -40.55287551879883,
+      "loss": 0.6886,
+      "losses/dpo": 0.6869601011276245,
+      "losses/sft": 1.2104542255401611,
+      "losses/total": 0.6869601011276245,
+      "ref_logps/chosen": -32.960693359375,
+      "ref_logps/rejected": -39.92742919921875,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.052775099873542786,
+      "rewards/margins": 0.009769486263394356,
+      "rewards/rejected": -0.06254458427429199,
+      "step": 41
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 5.9874098679402445,
+      "learning_rate": 4.97191011235955e-07,
+      "logps/chosen": -37.491756439208984,
+      "logps/rejected": -44.21824645996094,
+      "loss": 0.6903,
+      "losses/dpo": 0.6947405934333801,
+      "losses/sft": 1.5526431798934937,
+      "losses/total": 0.6947405934333801,
+      "ref_logps/chosen": -36.944496154785156,
+      "ref_logps/rejected": -43.608970642089844,
+      "rewards/accuracies": 0.53125,
+      "rewards/chosen": -0.05472607538104057,
+      "rewards/margins": 0.006201753858476877,
+      "rewards/rejected": -0.060927826911211014,
+      "step": 42
+    },
+    {
+      "epoch": 0.32,
+      "grad_norm": 5.890690107516997,
+      "learning_rate": 4.957865168539325e-07,
+      "logps/chosen": -37.96784210205078,
+      "logps/rejected": -44.18370056152344,
+      "loss": 0.6911,
+      "losses/dpo": 0.6906970143318176,
+      "losses/sft": 1.5630677938461304,
+      "losses/total": 0.6906970143318176,
+      "ref_logps/chosen": -37.31348419189453,
+      "ref_logps/rejected": -43.480613708496094,
+      "rewards/accuracies": 0.5078125,
+      "rewards/chosen": -0.06543563306331635,
+      "rewards/margins": 0.0048727355897426605,
+      "rewards/rejected": -0.07030836492776871,
+      "step": 43
+    },
+    {
+      "epoch": 0.33,
+      "grad_norm": 5.6620046922389475,
+      "learning_rate": 4.943820224719101e-07,
+      "logps/chosen": -33.37147903442383,
+      "logps/rejected": -40.975284576416016,
+      "loss": 0.6876,
+      "losses/dpo": 0.6959986090660095,
+      "losses/sft": 1.4914252758026123,
+      "losses/total": 0.6959986090660095,
+      "ref_logps/chosen": -32.77292251586914,
+      "ref_logps/rejected": -40.25656509399414,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.05985547974705696,
+      "rewards/margins": 0.012016610242426395,
+      "rewards/rejected": -0.07187209278345108,
+      "step": 44
+    },
+    {
+      "epoch": 0.34,
+      "grad_norm": 5.975010174130114,
+      "learning_rate": 4.929775280898877e-07,
+      "logps/chosen": -36.01771545410156,
+      "logps/rejected": -40.14152145385742,
+      "loss": 0.686,
+      "losses/dpo": 0.6887790560722351,
+      "losses/sft": 1.0922722816467285,
+      "losses/total": 0.6887790560722351,
+      "ref_logps/chosen": -35.33580017089844,
+      "ref_logps/rejected": -39.30628204345703,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.06819140911102295,
+      "rewards/margins": 0.015332860872149467,
+      "rewards/rejected": -0.08352427184581757,
+      "step": 45
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 5.941321729150391,
+      "learning_rate": 4.915730337078651e-07,
+      "logps/chosen": -37.23257064819336,
+      "logps/rejected": -43.88367462158203,
+      "loss": 0.6872,
+      "losses/dpo": 0.6943268775939941,
+      "losses/sft": 1.3947020769119263,
+      "losses/total": 0.6943268775939941,
+      "ref_logps/chosen": -36.47919845581055,
+      "ref_logps/rejected": -43.00157165527344,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.07533714175224304,
+      "rewards/margins": 0.012872692197561264,
+      "rewards/rejected": -0.08820983022451401,
+      "step": 46
+    },
+    {
+      "epoch": 0.35,
+      "grad_norm": 6.015115529321941,
+      "learning_rate": 4.901685393258427e-07,
+      "logps/chosen": -40.33525085449219,
+      "logps/rejected": -41.87712478637695,
+      "loss": 0.6904,
+      "losses/dpo": 0.6985194683074951,
+      "losses/sft": 1.4163267612457275,
+      "losses/total": 0.6985194683074951,
+      "ref_logps/chosen": -39.42625045776367,
+      "ref_logps/rejected": -40.897945404052734,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.09090035408735275,
+      "rewards/margins": 0.007017695810645819,
+      "rewards/rejected": -0.09791804850101471,
+      "step": 47
+    },
+    {
+      "epoch": 0.36,
+      "grad_norm": 5.733756968847646,
+      "learning_rate": 4.887640449438202e-07,
+      "logps/chosen": -36.97081756591797,
+      "logps/rejected": -42.80936050415039,
+      "loss": 0.6882,
+      "losses/dpo": 0.696724534034729,
+      "losses/sft": 1.2510000467300415,
+      "losses/total": 0.696724534034729,
+      "ref_logps/chosen": -36.11339569091797,
+      "ref_logps/rejected": -41.84064483642578,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.08574248850345612,
+      "rewards/margins": 0.011129248887300491,
+      "rewards/rejected": -0.09687173366546631,
+      "step": 48
+    },
+    {
+      "epoch": 0.37,
+      "grad_norm": 5.879327522594857,
+      "learning_rate": 4.873595505617978e-07,
+      "logps/chosen": -33.350772857666016,
+      "logps/rejected": -41.509918212890625,
+      "loss": 0.6838,
+      "losses/dpo": 0.6769124269485474,
+      "losses/sft": 1.218217372894287,
+      "losses/total": 0.6769124269485474,
+      "ref_logps/chosen": -32.432064056396484,
+      "ref_logps/rejected": -40.38986587524414,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.09187072515487671,
+      "rewards/margins": 0.020134272053837776,
+      "rewards/rejected": -0.11200500279664993,
+      "step": 49
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 6.158412418626052,
+      "learning_rate": 4.859550561797752e-07,
+      "logps/chosen": -38.05060577392578,
+      "logps/rejected": -45.274757385253906,
+      "loss": 0.6828,
+      "losses/dpo": 0.6935728788375854,
+      "losses/sft": 1.33810293674469,
+      "losses/total": 0.6935728788375854,
+      "ref_logps/chosen": -36.988319396972656,
+      "ref_logps/rejected": -43.98942565917969,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.10622845590114594,
+      "rewards/margins": 0.022304760292172432,
+      "rewards/rejected": -0.12853321433067322,
+      "step": 50
+    },
+    {
+      "epoch": 0.38,
+      "grad_norm": 6.285183993622672,
+      "learning_rate": 4.845505617977528e-07,
+      "logps/chosen": -38.64442443847656,
+      "logps/rejected": -42.03549575805664,
+      "loss": 0.6823,
+      "losses/dpo": 0.6838083267211914,
+      "losses/sft": 1.414647102355957,
+      "losses/total": 0.6838083267211914,
+      "ref_logps/chosen": -37.58879089355469,
+      "ref_logps/rejected": -40.74079132080078,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.1055637076497078,
+      "rewards/margins": 0.023906776681542397,
+      "rewards/rejected": -0.12947048246860504,
+      "step": 51
+    },
+    {
+      "epoch": 0.39,
+      "grad_norm": 5.8676763170916155,
+      "learning_rate": 4.831460674157303e-07,
+      "logps/chosen": -35.28524398803711,
+      "logps/rejected": -43.29574966430664,
+      "loss": 0.6824,
+      "losses/dpo": 0.6761789321899414,
+      "losses/sft": 1.1140950918197632,
+      "losses/total": 0.6761789321899414,
+      "ref_logps/chosen": -34.05834197998047,
+      "ref_logps/rejected": -41.828880310058594,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.12269000709056854,
+      "rewards/margins": 0.023997044190764427,
+      "rewards/rejected": -0.14668706059455872,
+      "step": 52
+    },
+    {
+      "epoch": 0.4,
+      "grad_norm": 6.153262147929733,
+      "learning_rate": 4.817415730337078e-07,
+      "logps/chosen": -34.02470016479492,
+      "logps/rejected": -38.51059341430664,
+      "loss": 0.6808,
+      "losses/dpo": 0.6883823871612549,
+      "losses/sft": 1.1792895793914795,
+      "losses/total": 0.6883823871612549,
+      "ref_logps/chosen": -32.95384216308594,
+      "ref_logps/rejected": -37.16828155517578,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.10708627104759216,
+      "rewards/margins": 0.02714475244283676,
+      "rewards/rejected": -0.13423103094100952,
+      "step": 53
+    },
+    {
+      "epoch": 0.41,
+      "grad_norm": 5.930275284806721,
+      "learning_rate": 4.803370786516854e-07,
+      "logps/chosen": -39.90019989013672,
+      "logps/rejected": -41.967960357666016,
+      "loss": 0.6817,
+      "losses/dpo": 0.6764520406723022,
+      "losses/sft": 1.4552464485168457,
+      "losses/total": 0.6764520406723022,
+      "ref_logps/chosen": -38.590492248535156,
+      "ref_logps/rejected": -40.406002044677734,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.1309707909822464,
+      "rewards/margins": 0.025225069373846054,
+      "rewards/rejected": -0.15619586408138275,
+      "step": 54
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 5.991852629106404,
+      "learning_rate": 4.789325842696629e-07,
+      "logps/chosen": -37.67607116699219,
+      "logps/rejected": -42.05184555053711,
+      "loss": 0.6906,
+      "losses/dpo": 0.6933637261390686,
+      "losses/sft": 1.3182023763656616,
+      "losses/total": 0.6933637261390686,
+      "ref_logps/chosen": -36.22807312011719,
+      "ref_logps/rejected": -40.539833068847656,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -0.14479960501194,
+      "rewards/margins": 0.006401616148650646,
+      "rewards/rejected": -0.15120121836662292,
+      "step": 55
+    },
+    {
+      "epoch": 0.42,
+      "grad_norm": 5.887970211048207,
+      "learning_rate": 4.775280898876405e-07,
+      "logps/chosen": -36.0313835144043,
+      "logps/rejected": -41.34480285644531,
+      "loss": 0.6806,
+      "losses/dpo": 0.6776463985443115,
+      "losses/sft": 1.3762413263320923,
+      "losses/total": 0.6776463985443115,
+      "ref_logps/chosen": -34.71417999267578,
+      "ref_logps/rejected": -39.74456024169922,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.131720170378685,
+      "rewards/margins": 0.02830405905842781,
+      "rewards/rejected": -0.1600242257118225,
+      "step": 56
+    },
+    {
+      "epoch": 0.43,
+      "grad_norm": 6.202121890129415,
+      "learning_rate": 4.7612359550561797e-07,
+      "logps/chosen": -39.138973236083984,
+      "logps/rejected": -44.62040710449219,
+      "loss": 0.6868,
+      "losses/dpo": 0.7154799699783325,
+      "losses/sft": 1.4311680793762207,
+      "losses/total": 0.7154799699783325,
+      "ref_logps/chosen": -37.48638153076172,
+      "ref_logps/rejected": -42.81147003173828,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.1652592122554779,
+      "rewards/margins": 0.01563437283039093,
+      "rewards/rejected": -0.18089357018470764,
+      "step": 57
+    },
+    {
+      "epoch": 0.44,
+      "grad_norm": 6.251681349620222,
+      "learning_rate": 4.747191011235955e-07,
+      "logps/chosen": -35.74232864379883,
+      "logps/rejected": -41.246910095214844,
+      "loss": 0.6762,
+      "losses/dpo": 0.6785226464271545,
+      "losses/sft": 1.2456488609313965,
+      "losses/total": 0.6785226464271545,
+      "ref_logps/chosen": -34.32046890258789,
+      "ref_logps/rejected": -39.45071792602539,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.14218537509441376,
+      "rewards/margins": 0.037433870136737823,
+      "rewards/rejected": -0.17961923778057098,
+      "step": 58
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 6.186489597098538,
+      "learning_rate": 4.7331460674157303e-07,
+      "logps/chosen": -38.993804931640625,
+      "logps/rejected": -48.68840789794922,
+      "loss": 0.6799,
+      "losses/dpo": 0.6576354503631592,
+      "losses/sft": 1.2577842473983765,
+      "losses/total": 0.6576354503631592,
+      "ref_logps/chosen": -37.452022552490234,
+      "ref_logps/rejected": -46.83843994140625,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.15417808294296265,
+      "rewards/margins": 0.03081856295466423,
+      "rewards/rejected": -0.18499664962291718,
+      "step": 59
+    },
+    {
+      "epoch": 0.45,
+      "grad_norm": 6.176384076659114,
+      "learning_rate": 4.7191011235955054e-07,
+      "logps/chosen": -36.94293975830078,
+      "logps/rejected": -43.75997543334961,
+      "loss": 0.6818,
+      "losses/dpo": 0.6777645349502563,
+      "losses/sft": 1.4646830558776855,
+      "losses/total": 0.6777645349502563,
+      "ref_logps/chosen": -35.410682678222656,
+      "ref_logps/rejected": -41.96450424194336,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.15322577953338623,
+      "rewards/margins": 0.026321690529584885,
+      "rewards/rejected": -0.17954745888710022,
+      "step": 60
+    },
+    {
+      "epoch": 0.46,
+      "grad_norm": 6.222192772165732,
+      "learning_rate": 4.705056179775281e-07,
+      "logps/chosen": -38.04816436767578,
+      "logps/rejected": -46.636329650878906,
+      "loss": 0.6813,
+      "losses/dpo": 0.6915292739868164,
+      "losses/sft": 1.5139144659042358,
+      "losses/total": 0.6915292739868164,
+      "ref_logps/chosen": -36.346763610839844,
+      "ref_logps/rejected": -44.65103530883789,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.170139878988266,
+      "rewards/margins": 0.028389303013682365,
+      "rewards/rejected": -0.1985291838645935,
+      "step": 61
+    },
+    {
+      "epoch": 0.47,
+      "grad_norm": 6.581489224854424,
+      "learning_rate": 4.691011235955056e-07,
+      "logps/chosen": -39.37269973754883,
+      "logps/rejected": -42.562713623046875,
+      "loss": 0.6729,
+      "losses/dpo": 0.6604301333427429,
+      "losses/sft": 1.2340155839920044,
+      "losses/total": 0.6604301333427429,
+      "ref_logps/chosen": -37.8352165222168,
+      "ref_logps/rejected": -40.57086181640625,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.15374788641929626,
+      "rewards/margins": 0.04543708637356758,
+      "rewards/rejected": -0.19918496906757355,
+      "step": 62
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 6.270954655004902,
+      "learning_rate": 4.6769662921348315e-07,
+      "logps/chosen": -35.57749938964844,
+      "logps/rejected": -43.94036102294922,
+      "loss": 0.6719,
+      "losses/dpo": 0.6689096689224243,
+      "losses/sft": 1.3980541229248047,
+      "losses/total": 0.6689096689224243,
+      "ref_logps/chosen": -33.870826721191406,
+      "ref_logps/rejected": -41.761024475097656,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.1706671416759491,
+      "rewards/margins": 0.0472659207880497,
+      "rewards/rejected": -0.2179330587387085,
+      "step": 63
+    },
+    {
+      "epoch": 0.48,
+      "grad_norm": 5.825145606750587,
+      "learning_rate": 4.662921348314606e-07,
+      "logps/chosen": -36.180145263671875,
+      "logps/rejected": -42.19972229003906,
+      "loss": 0.6867,
+      "losses/dpo": 0.6951602697372437,
+      "losses/sft": 1.4974910020828247,
+      "losses/total": 0.6951602697372437,
+      "ref_logps/chosen": -34.28754425048828,
+      "ref_logps/rejected": -40.13561248779297,
+      "rewards/accuracies": 0.5234375,
+      "rewards/chosen": -0.18926027417182922,
+      "rewards/margins": 0.017150741070508957,
+      "rewards/rejected": -0.20641100406646729,
+      "step": 64
+    },
+    {
+      "epoch": 0.49,
+      "grad_norm": 5.954970035091233,
+      "learning_rate": 4.6488764044943816e-07,
+      "logps/chosen": -41.472923278808594,
+      "logps/rejected": -45.73348617553711,
+      "loss": 0.6781,
+      "losses/dpo": 0.667303204536438,
+      "losses/sft": 1.494096040725708,
+      "losses/total": 0.667303204536438,
+      "ref_logps/chosen": -39.698944091796875,
+      "ref_logps/rejected": -43.6091423034668,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.177398219704628,
+      "rewards/margins": 0.03503631800413132,
+      "rewards/rejected": -0.2124345451593399,
+      "step": 65
+    },
+    {
+      "epoch": 0.5,
+      "grad_norm": 6.488401314246342,
+      "learning_rate": 4.634831460674157e-07,
+      "logps/chosen": -39.84260177612305,
+      "logps/rejected": -49.195159912109375,
+      "loss": 0.6716,
+      "losses/dpo": 0.6679590940475464,
+      "losses/sft": 1.3698948621749878,
+      "losses/total": 0.6679590940475464,
+      "ref_logps/chosen": -37.98674774169922,
+      "ref_logps/rejected": -46.86464309692383,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.18558543920516968,
+      "rewards/margins": 0.047466084361076355,
+      "rewards/rejected": -0.23305150866508484,
+      "step": 66
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 6.139969478930884,
+      "learning_rate": 4.620786516853932e-07,
+      "logps/chosen": -36.54951858520508,
+      "logps/rejected": -42.6442756652832,
+      "loss": 0.6689,
+      "losses/dpo": 0.650477409362793,
+      "losses/sft": 1.350743055343628,
+      "losses/total": 0.650477409362793,
+      "ref_logps/chosen": -34.77081298828125,
+      "ref_logps/rejected": -40.319297790527344,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.17787054181098938,
+      "rewards/margins": 0.05462724715471268,
+      "rewards/rejected": -0.23249778151512146,
+      "step": 67
+    },
+    {
+      "epoch": 0.51,
+      "grad_norm": 6.626542224506714,
+      "learning_rate": 4.606741573033708e-07,
+      "logps/chosen": -38.910194396972656,
+      "logps/rejected": -44.71943664550781,
+      "loss": 0.6588,
+      "losses/dpo": 0.6475476622581482,
+      "losses/sft": 1.0136208534240723,
+      "losses/total": 0.6475476622581482,
+      "ref_logps/chosen": -37.08655548095703,
+      "ref_logps/rejected": -42.12825393676758,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.18236377835273743,
+      "rewards/margins": 0.0767548531293869,
+      "rewards/rejected": -0.2591186463832855,
+      "step": 68
+    },
+    {
+      "epoch": 0.52,
+      "grad_norm": 6.316526648907962,
+      "learning_rate": 4.592696629213483e-07,
+      "logps/chosen": -39.12900924682617,
+      "logps/rejected": -47.94546890258789,
+      "loss": 0.6741,
+      "losses/dpo": 0.6746849417686462,
+      "losses/sft": 1.3253227472305298,
+      "losses/total": 0.6746849417686462,
+      "ref_logps/chosen": -36.78126525878906,
+      "ref_logps/rejected": -45.148887634277344,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.23477408289909363,
+      "rewards/margins": 0.044883839786052704,
+      "rewards/rejected": -0.27965790033340454,
+      "step": 69
+    },
+    {
+      "epoch": 0.53,
+      "grad_norm": 6.029340644383451,
+      "learning_rate": 4.5786516853932584e-07,
+      "logps/chosen": -37.168025970458984,
+      "logps/rejected": -43.3531494140625,
+      "loss": 0.6776,
+      "losses/dpo": 0.708085298538208,
+      "losses/sft": 1.549338698387146,
+      "losses/total": 0.708085298538208,
+      "ref_logps/chosen": -34.82072067260742,
+      "ref_logps/rejected": -40.60224151611328,
+      "rewards/accuracies": 0.578125,
+      "rewards/chosen": -0.2347305417060852,
+      "rewards/margins": 0.04036000370979309,
+      "rewards/rejected": -0.2750905454158783,
+      "step": 70
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 6.206457245959275,
+      "learning_rate": 4.5646067415730334e-07,
+      "logps/chosen": -37.381324768066406,
+      "logps/rejected": -44.06721878051758,
+      "loss": 0.667,
+      "losses/dpo": 0.6923149228096008,
+      "losses/sft": 1.499281883239746,
+      "losses/total": 0.6923149228096008,
+      "ref_logps/chosen": -35.004127502441406,
+      "ref_logps/rejected": -41.090354919433594,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.23771986365318298,
+      "rewards/margins": 0.05996667221188545,
+      "rewards/rejected": -0.29768651723861694,
+      "step": 71
+    },
+    {
+      "epoch": 0.54,
+      "grad_norm": 6.238705763349497,
+      "learning_rate": 4.550561797752809e-07,
+      "logps/chosen": -38.5302734375,
+      "logps/rejected": -48.384620666503906,
+      "loss": 0.6669,
+      "losses/dpo": 0.6794298887252808,
+      "losses/sft": 1.3331537246704102,
+      "losses/total": 0.6794298887252808,
+      "ref_logps/chosen": -35.968894958496094,
+      "ref_logps/rejected": -45.22618103027344,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.25613832473754883,
+      "rewards/margins": 0.05970541387796402,
+      "rewards/rejected": -0.31584370136260986,
+      "step": 72
+    },
+    {
+      "epoch": 0.55,
+      "grad_norm": 6.369104792363388,
+      "learning_rate": 4.536516853932584e-07,
+      "logps/chosen": -38.55243682861328,
+      "logps/rejected": -46.81627655029297,
+      "loss": 0.6609,
+      "losses/dpo": 0.6863117218017578,
+      "losses/sft": 1.404316782951355,
+      "losses/total": 0.6863117218017578,
+      "ref_logps/chosen": -35.96749496459961,
+      "ref_logps/rejected": -43.47722625732422,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.25849413871765137,
+      "rewards/margins": 0.07541059702634811,
+      "rewards/rejected": -0.3339047431945801,
+      "step": 73
+    },
+    {
+      "epoch": 0.56,
+      "grad_norm": 6.15339242747321,
+      "learning_rate": 4.522471910112359e-07,
+      "logps/chosen": -39.58115005493164,
+      "logps/rejected": -44.1653938293457,
+      "loss": 0.6782,
+      "losses/dpo": 0.7283678650856018,
+      "losses/sft": 1.3683419227600098,
+      "losses/total": 0.7283678650856018,
+      "ref_logps/chosen": -37.06593704223633,
+      "ref_logps/rejected": -41.25994873046875,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.25152140855789185,
+      "rewards/margins": 0.03902304172515869,
+      "rewards/rejected": -0.29054442048072815,
+      "step": 74
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 6.3073523395391105,
+      "learning_rate": 4.5084269662921347e-07,
+      "logps/chosen": -39.416324615478516,
+      "logps/rejected": -45.20884323120117,
+      "loss": 0.6702,
+      "losses/dpo": 0.6884989738464355,
+      "losses/sft": 1.2989376783370972,
+      "losses/total": 0.6884989738464355,
+      "ref_logps/chosen": -36.72044372558594,
+      "ref_logps/rejected": -41.92823791503906,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.2695878744125366,
+      "rewards/margins": 0.058472514152526855,
+      "rewards/rejected": -0.3280603885650635,
+      "step": 75
+    },
+    {
+      "epoch": 0.57,
+      "grad_norm": 6.353149418167083,
+      "learning_rate": 4.4943820224719097e-07,
+      "logps/chosen": -39.451934814453125,
+      "logps/rejected": -45.58893585205078,
+      "loss": 0.6758,
+      "losses/dpo": 0.6700998544692993,
+      "losses/sft": 1.423154354095459,
+      "losses/total": 0.6700998544692993,
+      "ref_logps/chosen": -36.6813850402832,
+      "ref_logps/rejected": -42.35654830932617,
+      "rewards/accuracies": 0.5546875,
+      "rewards/chosen": -0.2770548164844513,
+      "rewards/margins": 0.0461842380464077,
+      "rewards/rejected": -0.3232390582561493,
+      "step": 76
+    },
+    {
+      "epoch": 0.58,
+      "grad_norm": 6.243464205666771,
+      "learning_rate": 4.4803370786516853e-07,
+      "logps/chosen": -38.50192642211914,
+      "logps/rejected": -44.94510269165039,
+      "loss": 0.6654,
+      "losses/dpo": 0.6559799909591675,
+      "losses/sft": 1.3645029067993164,
+      "losses/total": 0.6559799909591675,
+      "ref_logps/chosen": -35.83762741088867,
+      "ref_logps/rejected": -41.636741638183594,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.2664298415184021,
+      "rewards/margins": 0.06440602242946625,
+      "rewards/rejected": -0.33083584904670715,
+      "step": 77
+    },
+    {
+      "epoch": 0.59,
+      "grad_norm": 6.529746915602167,
+      "learning_rate": 4.4662921348314603e-07,
+      "logps/chosen": -38.366634368896484,
+      "logps/rejected": -48.25501251220703,
+      "loss": 0.6701,
+      "losses/dpo": 0.6894055008888245,
+      "losses/sft": 1.4073951244354248,
+      "losses/total": 0.6894055008888245,
+      "ref_logps/chosen": -35.34041976928711,
+      "ref_logps/rejected": -44.6351318359375,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.3026217818260193,
+      "rewards/margins": 0.05936632677912712,
+      "rewards/rejected": -0.3619880974292755,
+      "step": 78
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 6.186041404774562,
+      "learning_rate": 4.452247191011236e-07,
+      "logps/chosen": -37.969024658203125,
+      "logps/rejected": -46.56663131713867,
+      "loss": 0.6582,
+      "losses/dpo": 0.6372844576835632,
+      "losses/sft": 1.1740036010742188,
+      "losses/total": 0.6372844576835632,
+      "ref_logps/chosen": -35.09920883178711,
+      "ref_logps/rejected": -42.874725341796875,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.2869817614555359,
+      "rewards/margins": 0.08220900595188141,
+      "rewards/rejected": -0.3691907525062561,
+      "step": 79
+    },
+    {
+      "epoch": 0.6,
+      "grad_norm": 6.41463598459145,
+      "learning_rate": 4.438202247191011e-07,
+      "logps/chosen": -43.05072021484375,
+      "logps/rejected": -49.178314208984375,
+      "loss": 0.656,
+      "losses/dpo": 0.6740515232086182,
+      "losses/sft": 1.4272187948226929,
+      "losses/total": 0.6740515232086182,
+      "ref_logps/chosen": -39.619014739990234,
+      "ref_logps/rejected": -44.83480453491211,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.3431706726551056,
+      "rewards/margins": 0.09118058532476425,
+      "rewards/rejected": -0.43435126543045044,
+      "step": 80
+    },
+    {
+      "epoch": 0.61,
+      "grad_norm": 6.840871211971457,
+      "learning_rate": 4.4241573033707865e-07,
+      "logps/chosen": -43.888370513916016,
+      "logps/rejected": -47.332916259765625,
+      "loss": 0.6729,
+      "losses/dpo": 0.666955828666687,
+      "losses/sft": 1.6874582767486572,
+      "losses/total": 0.666955828666687,
+      "ref_logps/chosen": -40.38330841064453,
+      "ref_logps/rejected": -43.3054084777832,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.350506067276001,
+      "rewards/margins": 0.05224461108446121,
+      "rewards/rejected": -0.4027506709098816,
+      "step": 81
+    },
+    {
+      "epoch": 0.62,
+      "grad_norm": 6.882624120223548,
+      "learning_rate": 4.410112359550562e-07,
+      "logps/chosen": -40.2237548828125,
+      "logps/rejected": -46.99496078491211,
+      "loss": 0.6607,
+      "losses/dpo": 0.6718687415122986,
+      "losses/sft": 1.5186784267425537,
+      "losses/total": 0.6718687415122986,
+      "ref_logps/chosen": -36.70365905761719,
+      "ref_logps/rejected": -42.63638687133789,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.3520098030567169,
+      "rewards/margins": 0.08384796977043152,
+      "rewards/rejected": -0.43585777282714844,
+      "step": 82
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 6.3488191331703385,
+      "learning_rate": 4.3960674157303366e-07,
+      "logps/chosen": -40.38496780395508,
+      "logps/rejected": -46.7673454284668,
+      "loss": 0.6535,
+      "losses/dpo": 0.6566940546035767,
+      "losses/sft": 1.3071130514144897,
+      "losses/total": 0.6566940546035767,
+      "ref_logps/chosen": -37.20966339111328,
+      "ref_logps/rejected": -42.63634490966797,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.3175300061702728,
+      "rewards/margins": 0.09556981176137924,
+      "rewards/rejected": -0.4130997955799103,
+      "step": 83
+    },
+    {
+      "epoch": 0.63,
+      "grad_norm": 6.624302993852389,
+      "learning_rate": 4.382022471910112e-07,
+      "logps/chosen": -42.17374801635742,
+      "logps/rejected": -49.17514419555664,
+      "loss": 0.6571,
+      "losses/dpo": 0.6181658506393433,
+      "losses/sft": 1.3204035758972168,
+      "losses/total": 0.6181658506393433,
+      "ref_logps/chosen": -38.54387664794922,
+      "ref_logps/rejected": -44.659461975097656,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.3629874587059021,
+      "rewards/margins": 0.08858054131269455,
+      "rewards/rejected": -0.45156803727149963,
+      "step": 84
+    },
+    {
+      "epoch": 0.64,
+      "grad_norm": 6.569663592940448,
+      "learning_rate": 4.367977528089887e-07,
+      "logps/chosen": -39.99671936035156,
+      "logps/rejected": -48.49413299560547,
+      "loss": 0.6562,
+      "losses/dpo": 0.6639370322227478,
+      "losses/sft": 1.6048388481140137,
+      "losses/total": 0.6639370322227478,
+      "ref_logps/chosen": -36.648887634277344,
+      "ref_logps/rejected": -44.24271774291992,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.3347826600074768,
+      "rewards/margins": 0.09035841375589371,
+      "rewards/rejected": -0.4251410961151123,
+      "step": 85
+    },
+    {
+      "epoch": 0.65,
+      "grad_norm": 6.597580499931281,
+      "learning_rate": 4.353932584269663e-07,
+      "logps/chosen": -41.4986572265625,
+      "logps/rejected": -48.67082214355469,
+      "loss": 0.6519,
+      "losses/dpo": 0.6654509902000427,
+      "losses/sft": 1.462377905845642,
+      "losses/total": 0.6654509902000427,
+      "ref_logps/chosen": -37.87129211425781,
+      "ref_logps/rejected": -43.97351837158203,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.3627370595932007,
+      "rewards/margins": 0.10699345916509628,
+      "rewards/rejected": -0.46973055601119995,
+      "step": 86
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 6.479183906632411,
+      "learning_rate": 4.339887640449438e-07,
+      "logps/chosen": -41.78961944580078,
+      "logps/rejected": -47.387901306152344,
+      "loss": 0.6791,
+      "losses/dpo": 0.6730961799621582,
+      "losses/sft": 1.1305738687515259,
+      "losses/total": 0.6730961799621582,
+      "ref_logps/chosen": -37.842323303222656,
+      "ref_logps/rejected": -42.93647766113281,
+      "rewards/accuracies": 0.515625,
+      "rewards/chosen": -0.3947296738624573,
+      "rewards/margins": 0.05041254311800003,
+      "rewards/rejected": -0.4451422691345215,
+      "step": 87
+    },
+    {
+      "epoch": 0.66,
+      "grad_norm": 6.926719176011086,
+      "learning_rate": 4.3258426966292134e-07,
+      "logps/chosen": -43.21299743652344,
+      "logps/rejected": -47.084434509277344,
+      "loss": 0.6673,
+      "losses/dpo": 0.6536482572555542,
+      "losses/sft": 1.2500860691070557,
+      "losses/total": 0.6536482572555542,
+      "ref_logps/chosen": -39.38795471191406,
+      "ref_logps/rejected": -42.582969665527344,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.38250401616096497,
+      "rewards/margins": 0.06764230877161026,
+      "rewards/rejected": -0.45014631748199463,
+      "step": 88
+    },
+    {
+      "epoch": 0.67,
+      "grad_norm": 6.563435223333862,
+      "learning_rate": 4.311797752808989e-07,
+      "logps/chosen": -40.25920104980469,
+      "logps/rejected": -49.489097595214844,
+      "loss": 0.6508,
+      "losses/dpo": 0.6591900587081909,
+      "losses/sft": 1.3429124355316162,
+      "losses/total": 0.6591900587081909,
+      "ref_logps/chosen": -36.316349029541016,
+      "ref_logps/rejected": -44.49497985839844,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3942852020263672,
+      "rewards/margins": 0.1051262766122818,
+      "rewards/rejected": -0.4994114637374878,
+      "step": 89
+    },
+    {
+      "epoch": 0.68,
+      "grad_norm": 6.671599802331672,
+      "learning_rate": 4.297752808988764e-07,
+      "logps/chosen": -42.98493957519531,
+      "logps/rejected": -45.29029846191406,
+      "loss": 0.6775,
+      "losses/dpo": 0.6989056468009949,
+      "losses/sft": 1.7236398458480835,
+      "losses/total": 0.6989056468009949,
+      "ref_logps/chosen": -38.74510955810547,
+      "ref_logps/rejected": -40.55065155029297,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.4239833354949951,
+      "rewards/margins": 0.04998103156685829,
+      "rewards/rejected": -0.4739643633365631,
+      "step": 90
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 6.921605107059482,
+      "learning_rate": 4.2837078651685396e-07,
+      "logps/chosen": -42.04779052734375,
+      "logps/rejected": -47.75447463989258,
+      "loss": 0.6577,
+      "losses/dpo": 0.6272084712982178,
+      "losses/sft": 1.5017703771591187,
+      "losses/total": 0.6272084712982178,
+      "ref_logps/chosen": -37.97248458862305,
+      "ref_logps/rejected": -42.7065315246582,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.40753045678138733,
+      "rewards/margins": 0.09726397693157196,
+      "rewards/rejected": -0.5047944784164429,
+      "step": 91
+    },
+    {
+      "epoch": 0.69,
+      "grad_norm": 6.759776662372792,
+      "learning_rate": 4.269662921348314e-07,
+      "logps/chosen": -44.38639831542969,
+      "logps/rejected": -53.21236038208008,
+      "loss": 0.6431,
+      "losses/dpo": 0.7165791988372803,
+      "losses/sft": 1.5609912872314453,
+      "losses/total": 0.7165791988372803,
+      "ref_logps/chosen": -39.95743942260742,
+      "ref_logps/rejected": -47.561588287353516,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.4428956210613251,
+      "rewards/margins": 0.122181735932827,
+      "rewards/rejected": -0.5650773644447327,
+      "step": 92
+    },
+    {
+      "epoch": 0.7,
+      "grad_norm": 6.767540666000382,
+      "learning_rate": 4.2556179775280896e-07,
+      "logps/chosen": -39.6769905090332,
+      "logps/rejected": -45.86317443847656,
+      "loss": 0.6772,
+      "losses/dpo": 0.6330491900444031,
+      "losses/sft": 1.33146333694458,
+      "losses/total": 0.6330491900444031,
+      "ref_logps/chosen": -35.39988708496094,
+      "ref_logps/rejected": -41.038116455078125,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.42771056294441223,
+      "rewards/margins": 0.05479476973414421,
+      "rewards/rejected": -0.4825053811073303,
+      "step": 93
+    },
+    {
+      "epoch": 0.71,
+      "grad_norm": 6.521925535129618,
+      "learning_rate": 4.2415730337078647e-07,
+      "logps/chosen": -43.78227996826172,
+      "logps/rejected": -47.82459259033203,
+      "loss": 0.6607,
+      "losses/dpo": 0.6985595226287842,
+      "losses/sft": 1.530924677848816,
+      "losses/total": 0.6985595226287842,
+      "ref_logps/chosen": -39.28633499145508,
+      "ref_logps/rejected": -42.36700439453125,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.4495944082736969,
+      "rewards/margins": 0.09616444259881973,
+      "rewards/rejected": -0.545758843421936,
+      "step": 94
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 6.606025652021848,
+      "learning_rate": 4.22752808988764e-07,
+      "logps/chosen": -44.53917694091797,
+      "logps/rejected": -49.32555389404297,
+      "loss": 0.6783,
+      "losses/dpo": 0.6152039766311646,
+      "losses/sft": 1.5025076866149902,
+      "losses/total": 0.6152039766311646,
+      "ref_logps/chosen": -39.66822814941406,
+      "ref_logps/rejected": -43.90290451049805,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.4870951175689697,
+      "rewards/margins": 0.05516959726810455,
+      "rewards/rejected": -0.5422646999359131,
+      "step": 95
+    },
+    {
+      "epoch": 0.72,
+      "grad_norm": 6.456362063653043,
+      "learning_rate": 4.2134831460674153e-07,
+      "logps/chosen": -40.66051483154297,
+      "logps/rejected": -50.35266876220703,
+      "loss": 0.6339,
+      "losses/dpo": 0.5940225124359131,
+      "losses/sft": 1.3329205513000488,
+      "losses/total": 0.5940225124359131,
+      "ref_logps/chosen": -36.583961486816406,
+      "ref_logps/rejected": -44.78839874267578,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.40765535831451416,
+      "rewards/margins": 0.14877161383628845,
+      "rewards/rejected": -0.556427001953125,
+      "step": 96
+    },
+    {
+      "epoch": 0.73,
+      "grad_norm": 7.002732175851258,
+      "learning_rate": 4.199438202247191e-07,
+      "logps/chosen": -40.17961502075195,
+      "logps/rejected": -50.040138244628906,
+      "loss": 0.6253,
+      "losses/dpo": 0.597855806350708,
+      "losses/sft": 1.5503275394439697,
+      "losses/total": 0.597855806350708,
+      "ref_logps/chosen": -35.91657257080078,
+      "ref_logps/rejected": -44.05973815917969,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.426303893327713,
+      "rewards/margins": 0.17173629999160767,
+      "rewards/rejected": -0.5980401635169983,
+      "step": 97
+    },
+    {
+      "epoch": 0.74,
+      "grad_norm": 6.7690281568226345,
+      "learning_rate": 4.1853932584269664e-07,
+      "logps/chosen": -43.26731872558594,
+      "logps/rejected": -48.155426025390625,
+      "loss": 0.6528,
+      "losses/dpo": 0.6972070932388306,
+      "losses/sft": 1.3802154064178467,
+      "losses/total": 0.6972070932388306,
+      "ref_logps/chosen": -38.819725036621094,
+      "ref_logps/rejected": -42.62653350830078,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.4447590410709381,
+      "rewards/margins": 0.10813023149967194,
+      "rewards/rejected": -0.5528892278671265,
+      "step": 98
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 6.184786618255584,
+      "learning_rate": 4.1713483146067415e-07,
+      "logps/chosen": -39.052734375,
+      "logps/rejected": -45.65272521972656,
+      "loss": 0.6289,
+      "losses/dpo": 0.5835955142974854,
+      "losses/sft": 1.2479004859924316,
+      "losses/total": 0.5835955142974854,
+      "ref_logps/chosen": -35.05288314819336,
+      "ref_logps/rejected": -40.07136154174805,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.39998501539230347,
+      "rewards/margins": 0.15815110504627228,
+      "rewards/rejected": -0.5581361055374146,
+      "step": 99
+    },
+    {
+      "epoch": 0.75,
+      "grad_norm": 6.925410262368385,
+      "learning_rate": 4.157303370786517e-07,
+      "logps/chosen": -44.249752044677734,
+      "logps/rejected": -44.935245513916016,
+      "loss": 0.6711,
+      "losses/dpo": 0.5667402744293213,
+      "losses/sft": 1.424223780632019,
+      "losses/total": 0.5667402744293213,
+      "ref_logps/chosen": -38.972206115722656,
+      "ref_logps/rejected": -38.912513732910156,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.5277543067932129,
+      "rewards/margins": 0.07451874017715454,
+      "rewards/rejected": -0.6022731065750122,
+      "step": 100
+    },
+    {
+      "epoch": 0.76,
+      "grad_norm": 6.586928303266985,
+      "learning_rate": 4.1432584269662915e-07,
+      "logps/chosen": -39.689693450927734,
+      "logps/rejected": -48.46234130859375,
+      "loss": 0.6509,
+      "losses/dpo": 0.6659662127494812,
+      "losses/sft": 1.3264880180358887,
+      "losses/total": 0.6659662127494812,
+      "ref_logps/chosen": -35.252235412597656,
+      "ref_logps/rejected": -42.87641525268555,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.4437457323074341,
+      "rewards/margins": 0.11484652757644653,
+      "rewards/rejected": -0.5585922598838806,
+      "step": 101
+    },
+    {
+      "epoch": 0.77,
+      "grad_norm": 6.103700351208487,
+      "learning_rate": 4.129213483146067e-07,
+      "logps/chosen": -38.51823425292969,
+      "logps/rejected": -43.52346420288086,
+      "loss": 0.6509,
+      "losses/dpo": 0.6539372205734253,
+      "losses/sft": 1.3750677108764648,
+      "losses/total": 0.6539372205734253,
+      "ref_logps/chosen": -34.18145751953125,
+      "ref_logps/rejected": -38.04938507080078,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.4336775541305542,
+      "rewards/margins": 0.11373014003038406,
+      "rewards/rejected": -0.5474076867103577,
+      "step": 102
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 6.228970412657457,
+      "learning_rate": 4.115168539325842e-07,
+      "logps/chosen": -42.1187629699707,
+      "logps/rejected": -47.93737030029297,
+      "loss": 0.6451,
+      "losses/dpo": 0.646047055721283,
+      "losses/sft": 1.497565507888794,
+      "losses/total": 0.646047055721283,
+      "ref_logps/chosen": -37.47208786010742,
+      "ref_logps/rejected": -42.03216552734375,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.4646672010421753,
+      "rewards/margins": 0.12585340440273285,
+      "rewards/rejected": -0.5905206203460693,
+      "step": 103
+    },
+    {
+      "epoch": 0.78,
+      "grad_norm": 6.892504983818195,
+      "learning_rate": 4.1011235955056177e-07,
+      "logps/chosen": -42.86591339111328,
+      "logps/rejected": -48.31887435913086,
+      "loss": 0.6494,
+      "losses/dpo": 0.6817602515220642,
+      "losses/sft": 1.5651347637176514,
+      "losses/total": 0.6817602515220642,
+      "ref_logps/chosen": -37.89812088012695,
+      "ref_logps/rejected": -42.230018615722656,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.4967789351940155,
+      "rewards/margins": 0.11210669577121735,
+      "rewards/rejected": -0.608885645866394,
+      "step": 104
+    },
+    {
+      "epoch": 0.79,
+      "grad_norm": 6.344274461611194,
+      "learning_rate": 4.0870786516853933e-07,
+      "logps/chosen": -38.07393264770508,
+      "logps/rejected": -46.695167541503906,
+      "loss": 0.6485,
+      "losses/dpo": 0.6693782806396484,
+      "losses/sft": 1.4964573383331299,
+      "losses/total": 0.6693782806396484,
+      "ref_logps/chosen": -33.26963806152344,
+      "ref_logps/rejected": -40.731658935546875,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.48042935132980347,
+      "rewards/margins": 0.11592163890600204,
+      "rewards/rejected": -0.5963510274887085,
+      "step": 105
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 6.340224167086584,
+      "learning_rate": 4.0730337078651683e-07,
+      "logps/chosen": -34.64811706542969,
+      "logps/rejected": -44.656005859375,
+      "loss": 0.6748,
+      "losses/dpo": 0.7200191020965576,
+      "losses/sft": 1.2917957305908203,
+      "losses/total": 0.7200191020965576,
+      "ref_logps/chosen": -29.991100311279297,
+      "ref_logps/rejected": -39.274078369140625,
+      "rewards/accuracies": 0.609375,
+      "rewards/chosen": -0.4657020568847656,
+      "rewards/margins": 0.07249079644680023,
+      "rewards/rejected": -0.5381928086280823,
+      "step": 106
+    },
+    {
+      "epoch": 0.81,
+      "grad_norm": 6.810734673228144,
+      "learning_rate": 4.058988764044944e-07,
+      "logps/chosen": -43.92599868774414,
+      "logps/rejected": -51.205841064453125,
+      "loss": 0.6618,
+      "losses/dpo": 0.7375708818435669,
+      "losses/sft": 1.6257060766220093,
+      "losses/total": 0.7375708818435669,
+      "ref_logps/chosen": -38.43563461303711,
+      "ref_logps/rejected": -44.785430908203125,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.5490366220474243,
+      "rewards/margins": 0.09300415217876434,
+      "rewards/rejected": -0.6420407295227051,
+      "step": 107
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 6.646468325900178,
+      "learning_rate": 4.044943820224719e-07,
+      "logps/chosen": -41.907615661621094,
+      "logps/rejected": -47.17523956298828,
+      "loss": 0.6665,
+      "losses/dpo": 0.6535848379135132,
+      "losses/sft": 1.5487432479858398,
+      "losses/total": 0.6535848379135132,
+      "ref_logps/chosen": -36.636451721191406,
+      "ref_logps/rejected": -41.005767822265625,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5271163582801819,
+      "rewards/margins": 0.08983068913221359,
+      "rewards/rejected": -0.6169470548629761,
+      "step": 108
+    },
+    {
+      "epoch": 0.82,
+      "grad_norm": 6.796059919133426,
+      "learning_rate": 4.0308988764044945e-07,
+      "logps/chosen": -43.95292663574219,
+      "logps/rejected": -48.59518814086914,
+      "loss": 0.6619,
+      "losses/dpo": 0.6743461489677429,
+      "losses/sft": 1.5721744298934937,
+      "losses/total": 0.6743461489677429,
+      "ref_logps/chosen": -38.58289337158203,
+      "ref_logps/rejected": -42.209529876708984,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.5370036363601685,
+      "rewards/margins": 0.10156210511922836,
+      "rewards/rejected": -0.6385657787322998,
+      "step": 109
+    },
+    {
+      "epoch": 0.83,
+      "grad_norm": 6.472585584476915,
+      "learning_rate": 4.0168539325842696e-07,
+      "logps/chosen": -40.216651916503906,
+      "logps/rejected": -45.985801696777344,
+      "loss": 0.6793,
+      "losses/dpo": 0.7015002965927124,
+      "losses/sft": 1.661520004272461,
+      "losses/total": 0.7015002965927124,
+      "ref_logps/chosen": -34.707008361816406,
+      "ref_logps/rejected": -39.90086364746094,
+      "rewards/accuracies": 0.5859375,
+      "rewards/chosen": -0.5509647130966187,
+      "rewards/margins": 0.05752916634082794,
+      "rewards/rejected": -0.6084938049316406,
+      "step": 110
+    },
+    {
+      "epoch": 0.84,
+      "grad_norm": 6.441943056630329,
+      "learning_rate": 4.0028089887640446e-07,
+      "logps/chosen": -40.84614944458008,
+      "logps/rejected": -49.78240966796875,
+      "loss": 0.6495,
+      "losses/dpo": 0.6826507449150085,
+      "losses/sft": 1.6292600631713867,
+      "losses/total": 0.6826507449150085,
+      "ref_logps/chosen": -35.30500411987305,
+      "ref_logps/rejected": -43.05198669433594,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.554114580154419,
+      "rewards/margins": 0.1189279854297638,
+      "rewards/rejected": -0.6730425953865051,
+      "step": 111
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 6.739697461780844,
+      "learning_rate": 3.9887640449438196e-07,
+      "logps/chosen": -43.101287841796875,
+      "logps/rejected": -51.60324478149414,
+      "loss": 0.6365,
+      "losses/dpo": 0.617262601852417,
+      "losses/sft": 1.4229466915130615,
+      "losses/total": 0.617262601852417,
+      "ref_logps/chosen": -37.866512298583984,
+      "ref_logps/rejected": -44.80317687988281,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.52347731590271,
+      "rewards/margins": 0.1565295159816742,
+      "rewards/rejected": -0.6800068020820618,
+      "step": 112
+    },
+    {
+      "epoch": 0.85,
+      "grad_norm": 7.023593025692052,
+      "learning_rate": 3.974719101123595e-07,
+      "logps/chosen": -41.23580551147461,
+      "logps/rejected": -52.932403564453125,
+      "loss": 0.621,
+      "losses/dpo": 0.6078984141349792,
+      "losses/sft": 1.4510893821716309,
+      "losses/total": 0.6078984141349792,
+      "ref_logps/chosen": -36.275489807128906,
+      "ref_logps/rejected": -46.033939361572266,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.49603164196014404,
+      "rewards/margins": 0.19381484389305115,
+      "rewards/rejected": -0.6898465156555176,
+      "step": 113
+    },
+    {
+      "epoch": 0.86,
+      "grad_norm": 6.441040652132362,
+      "learning_rate": 3.960674157303371e-07,
+      "logps/chosen": -38.938751220703125,
+      "logps/rejected": -47.65938186645508,
+      "loss": 0.6436,
+      "losses/dpo": 0.6575403809547424,
+      "losses/sft": 1.4100581407546997,
+      "losses/total": 0.6575403809547424,
+      "ref_logps/chosen": -33.69282531738281,
+      "ref_logps/rejected": -41.06393814086914,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.5245928764343262,
+      "rewards/margins": 0.13495120406150818,
+      "rewards/rejected": -0.6595441102981567,
+      "step": 114
+    },
+    {
+      "epoch": 0.87,
+      "grad_norm": 6.831582112977574,
+      "learning_rate": 3.946629213483146e-07,
+      "logps/chosen": -41.802799224853516,
+      "logps/rejected": -49.96432876586914,
+      "loss": 0.6368,
+      "losses/dpo": 0.6237789392471313,
+      "losses/sft": 1.5177757740020752,
+      "losses/total": 0.6237789392471313,
+      "ref_logps/chosen": -36.240840911865234,
+      "ref_logps/rejected": -42.76991653442383,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.5561960935592651,
+      "rewards/margins": 0.16324520111083984,
+      "rewards/rejected": -0.719441294670105,
+      "step": 115
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 7.410431235906546,
+      "learning_rate": 3.9325842696629214e-07,
+      "logps/chosen": -45.350669860839844,
+      "logps/rejected": -48.64668655395508,
+      "loss": 0.6848,
+      "losses/dpo": 0.746525228023529,
+      "losses/sft": 1.8295865058898926,
+      "losses/total": 0.746525228023529,
+      "ref_logps/chosen": -39.40565490722656,
+      "ref_logps/rejected": -42.114837646484375,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.5945014357566833,
+      "rewards/margins": 0.058684106916189194,
+      "rewards/rejected": -0.6531856060028076,
+      "step": 116
+    },
+    {
+      "epoch": 0.88,
+      "grad_norm": 6.7521875642568245,
+      "learning_rate": 3.9185393258426964e-07,
+      "logps/chosen": -42.558738708496094,
+      "logps/rejected": -48.05232238769531,
+      "loss": 0.6395,
+      "losses/dpo": 0.6771230697631836,
+      "losses/sft": 1.4978280067443848,
+      "losses/total": 0.6771230697631836,
+      "ref_logps/chosen": -37.11561584472656,
+      "ref_logps/rejected": -41.03240203857422,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.5443119406700134,
+      "rewards/margins": 0.1576804369688034,
+      "rewards/rejected": -0.701992392539978,
+      "step": 117
+    },
+    {
+      "epoch": 0.89,
+      "grad_norm": 6.852926877450363,
+      "learning_rate": 3.904494382022472e-07,
+      "logps/chosen": -43.11158752441406,
+      "logps/rejected": -50.49040985107422,
+      "loss": 0.6294,
+      "losses/dpo": 0.6048296689987183,
+      "losses/sft": 1.4263670444488525,
+      "losses/total": 0.6048296689987183,
+      "ref_logps/chosen": -37.525428771972656,
+      "ref_logps/rejected": -43.2536735534668,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.5586156845092773,
+      "rewards/margins": 0.16505761444568634,
+      "rewards/rejected": -0.7236733436584473,
+      "step": 118
+    },
+    {
+      "epoch": 0.9,
+      "grad_norm": 7.482804237101483,
+      "learning_rate": 3.890449438202247e-07,
+      "logps/chosen": -42.81207275390625,
+      "logps/rejected": -48.305213928222656,
+      "loss": 0.6723,
+      "losses/dpo": 0.6438789367675781,
+      "losses/sft": 1.3842287063598633,
+      "losses/total": 0.6438789367675781,
+      "ref_logps/chosen": -37.165802001953125,
+      "ref_logps/rejected": -41.8455696105957,
+      "rewards/accuracies": 0.5625,
+      "rewards/chosen": -0.5646266937255859,
+      "rewards/margins": 0.08133774995803833,
+      "rewards/rejected": -0.6459644436836243,
+      "step": 119
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 6.612121590764,
+      "learning_rate": 3.876404494382022e-07,
+      "logps/chosen": -40.42414093017578,
+      "logps/rejected": -49.36077880859375,
+      "loss": 0.6449,
+      "losses/dpo": 0.6813696622848511,
+      "losses/sft": 1.6905653476715088,
+      "losses/total": 0.6813696622848511,
+      "ref_logps/chosen": -34.49721145629883,
+      "ref_logps/rejected": -42.062259674072266,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.5926928520202637,
+      "rewards/margins": 0.13715943694114685,
+      "rewards/rejected": -0.7298523187637329,
+      "step": 120
+    },
+    {
+      "epoch": 0.91,
+      "grad_norm": 6.529474424321076,
+      "learning_rate": 3.8623595505617977e-07,
+      "logps/chosen": -43.11798095703125,
+      "logps/rejected": -51.801422119140625,
+      "loss": 0.6396,
+      "losses/dpo": 0.6661785840988159,
+      "losses/sft": 1.4052226543426514,
+      "losses/total": 0.6661785840988159,
+      "ref_logps/chosen": -37.71819305419922,
+      "ref_logps/rejected": -44.816795349121094,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.5399786233901978,
+      "rewards/margins": 0.15848389267921448,
+      "rewards/rejected": -0.6984626054763794,
+      "step": 121
+    },
+    {
+      "epoch": 0.92,
+      "grad_norm": 6.759925517450978,
+      "learning_rate": 3.8483146067415727e-07,
+      "logps/chosen": -44.635955810546875,
+      "logps/rejected": -53.689002990722656,
+      "loss": 0.6172,
+      "losses/dpo": 0.5884010195732117,
+      "losses/sft": 1.7550606727600098,
+      "losses/total": 0.5884010195732117,
+      "ref_logps/chosen": -38.62323760986328,
+      "ref_logps/rejected": -45.53942108154297,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6012718677520752,
+      "rewards/margins": 0.21368616819381714,
+      "rewards/rejected": -0.8149580359458923,
+      "step": 122
+    },
+    {
+      "epoch": 0.93,
+      "grad_norm": 6.5150883611431,
+      "learning_rate": 3.834269662921348e-07,
+      "logps/chosen": -41.34044647216797,
+      "logps/rejected": -50.817466735839844,
+      "loss": 0.6373,
+      "losses/dpo": 0.6241766214370728,
+      "losses/sft": 1.3161594867706299,
+      "losses/total": 0.6241766214370728,
+      "ref_logps/chosen": -35.719482421875,
+      "ref_logps/rejected": -43.593727111816406,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.562096118927002,
+      "rewards/margins": 0.16027754545211792,
+      "rewards/rejected": -0.7223736047744751,
+      "step": 123
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 7.117780641865591,
+      "learning_rate": 3.8202247191011233e-07,
+      "logps/chosen": -40.98164367675781,
+      "logps/rejected": -46.440032958984375,
+      "loss": 0.6259,
+      "losses/dpo": 0.7275031805038452,
+      "losses/sft": 1.3312557935714722,
+      "losses/total": 0.7275031805038452,
+      "ref_logps/chosen": -36.18989181518555,
+      "ref_logps/rejected": -39.75829315185547,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.4791754186153412,
+      "rewards/margins": 0.1889985203742981,
+      "rewards/rejected": -0.6681739091873169,
+      "step": 124
+    },
+    {
+      "epoch": 0.94,
+      "grad_norm": 6.810891199884393,
+      "learning_rate": 3.806179775280899e-07,
+      "logps/chosen": -45.05056381225586,
+      "logps/rejected": -51.01411819458008,
+      "loss": 0.6422,
+      "losses/dpo": 0.5984268188476562,
+      "losses/sft": 1.6079349517822266,
+      "losses/total": 0.5984268188476562,
+      "ref_logps/chosen": -39.050132751464844,
+      "ref_logps/rejected": -43.51178741455078,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.6000430583953857,
+      "rewards/margins": 0.15018987655639648,
+      "rewards/rejected": -0.750232994556427,
+      "step": 125
+    },
+    {
+      "epoch": 0.95,
+      "grad_norm": 7.000300157233934,
+      "learning_rate": 3.792134831460674e-07,
+      "logps/chosen": -45.488685607910156,
+      "logps/rejected": -53.24082946777344,
+      "loss": 0.6293,
+      "losses/dpo": 0.6512585878372192,
+      "losses/sft": 1.70095694065094,
+      "losses/total": 0.6512585878372192,
+      "ref_logps/chosen": -39.463134765625,
+      "ref_logps/rejected": -45.5393180847168,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.602555513381958,
+      "rewards/margins": 0.16759565472602844,
+      "rewards/rejected": -0.7701511383056641,
+      "step": 126
+    },
+    {
+      "epoch": 0.96,
+      "grad_norm": 6.866553634275337,
+      "learning_rate": 3.7780898876404495e-07,
+      "logps/chosen": -46.352333068847656,
+      "logps/rejected": -50.944129943847656,
+      "loss": 0.6501,
+      "losses/dpo": 0.7261393070220947,
+      "losses/sft": 1.7794756889343262,
+      "losses/total": 0.7261393070220947,
+      "ref_logps/chosen": -40.64281463623047,
+      "ref_logps/rejected": -43.911109924316406,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.5709517002105713,
+      "rewards/margins": 0.13234999775886536,
+      "rewards/rejected": -0.7033016681671143,
+      "step": 127
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 7.1746623953368385,
+      "learning_rate": 3.7640449438202245e-07,
+      "logps/chosen": -42.81266403198242,
+      "logps/rejected": -47.58647155761719,
+      "loss": 0.6484,
+      "losses/dpo": 0.6509548425674438,
+      "losses/sft": 1.4893585443496704,
+      "losses/total": 0.6509548425674438,
+      "ref_logps/chosen": -36.92261505126953,
+      "ref_logps/rejected": -40.24052429199219,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.5890049338340759,
+      "rewards/margins": 0.14558979868888855,
+      "rewards/rejected": -0.7345947623252869,
+      "step": 128
+    },
+    {
+      "epoch": 0.97,
+      "grad_norm": 6.639733312989805,
+      "learning_rate": 3.75e-07,
+      "logps/chosen": -42.699241638183594,
+      "logps/rejected": -49.38917922973633,
+      "loss": 0.6384,
+      "losses/dpo": 0.7398217916488647,
+      "losses/sft": 1.8203296661376953,
+      "losses/total": 0.7398217916488647,
+      "ref_logps/chosen": -36.63134002685547,
+      "ref_logps/rejected": -41.697608947753906,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.6067899465560913,
+      "rewards/margins": 0.16236720979213715,
+      "rewards/rejected": -0.7691571712493896,
+      "step": 129
+    },
+    {
+      "epoch": 0.98,
+      "grad_norm": 6.840872832592906,
+      "learning_rate": 3.735955056179775e-07,
+      "logps/chosen": -38.55268096923828,
+      "logps/rejected": -46.57276153564453,
+      "loss": 0.6455,
+      "losses/dpo": 0.6873192191123962,
+      "losses/sft": 1.3916716575622559,
+      "losses/total": 0.6873192191123962,
+      "ref_logps/chosen": -33.59415054321289,
+      "ref_logps/rejected": -40.15166091918945,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.4958529472351074,
+      "rewards/margins": 0.14625714719295502,
+      "rewards/rejected": -0.6421101093292236,
+      "step": 130
+    },
+    {
+      "epoch": 0.99,
+      "grad_norm": 7.001016051342946,
+      "learning_rate": 3.72191011235955e-07,
+      "logps/chosen": -42.6187629699707,
+      "logps/rejected": -47.61820983886719,
+      "loss": 0.6507,
+      "losses/dpo": 0.555785059928894,
+      "losses/sft": 1.6289881467819214,
+      "losses/total": 0.555785059928894,
+      "ref_logps/chosen": -36.840423583984375,
+      "ref_logps/rejected": -40.43097686767578,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.5778340101242065,
+      "rewards/margins": 0.14088886976242065,
+      "rewards/rejected": -0.7187228202819824,
+      "step": 131
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 6.69674019742392,
+      "learning_rate": 3.707865168539326e-07,
+      "logps/chosen": -42.53615951538086,
+      "logps/rejected": -50.24591064453125,
+      "loss": 0.639,
+      "losses/dpo": 0.658089280128479,
+      "losses/sft": 1.6500622034072876,
+      "losses/total": 0.658089280128479,
+      "ref_logps/chosen": -36.53786087036133,
+      "ref_logps/rejected": -42.35614776611328,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.599829912185669,
+      "rewards/margins": 0.1891460418701172,
+      "rewards/rejected": -0.7889760136604309,
+      "step": 132
+    },
+    {
+      "epoch": 1.0,
+      "grad_norm": 6.641666012117393,
+      "learning_rate": 3.693820224719101e-07,
+      "logps/chosen": -40.87996292114258,
+      "logps/rejected": -51.007049560546875,
+      "loss": 0.6244,
+      "losses/dpo": 0.6069691181182861,
+      "losses/sft": 1.3164952993392944,
+      "losses/total": 0.6069691181182861,
+      "ref_logps/chosen": -35.29632568359375,
+      "ref_logps/rejected": -43.411521911621094,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.5583640933036804,
+      "rewards/margins": 0.20118848979473114,
+      "rewards/rejected": -0.7595525979995728,
+      "step": 133
+    },
+    {
+      "epoch": 1.01,
+      "grad_norm": 6.588946335066842,
+      "learning_rate": 3.6797752808988764e-07,
+      "logps/chosen": -42.399169921875,
+      "logps/rejected": -51.85491180419922,
+      "loss": 0.6191,
+      "losses/dpo": 0.5947903394699097,
+      "losses/sft": 1.4862775802612305,
+      "losses/total": 0.5947903394699097,
+      "ref_logps/chosen": -36.425140380859375,
+      "ref_logps/rejected": -43.687225341796875,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.5974029898643494,
+      "rewards/margins": 0.2193659394979477,
+      "rewards/rejected": -0.8167688846588135,
+      "step": 134
+    },
+    {
+      "epoch": 1.02,
+      "grad_norm": 6.690035072129239,
+      "learning_rate": 3.6657303370786514e-07,
+      "logps/chosen": -41.394432067871094,
+      "logps/rejected": -53.464168548583984,
+      "loss": 0.6,
+      "losses/dpo": 0.6503059267997742,
+      "losses/sft": 1.5431207418441772,
+      "losses/total": 0.6503059267997742,
+      "ref_logps/chosen": -35.42436218261719,
+      "ref_logps/rejected": -44.949180603027344,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5970069766044617,
+      "rewards/margins": 0.25449231266975403,
+      "rewards/rejected": -0.8514993190765381,
+      "step": 135
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 7.159913988692313,
+      "learning_rate": 3.651685393258427e-07,
+      "logps/chosen": -47.11418533325195,
+      "logps/rejected": -49.349937438964844,
+      "loss": 0.6642,
+      "losses/dpo": 0.70440673828125,
+      "losses/sft": 1.6172586679458618,
+      "losses/total": 0.70440673828125,
+      "ref_logps/chosen": -40.66786193847656,
+      "ref_logps/rejected": -41.70207977294922,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.6446323394775391,
+      "rewards/margins": 0.12015305459499359,
+      "rewards/rejected": -0.7647854685783386,
+      "step": 136
+    },
+    {
+      "epoch": 1.03,
+      "grad_norm": 6.861296978626527,
+      "learning_rate": 3.637640449438202e-07,
+      "logps/chosen": -41.73448181152344,
+      "logps/rejected": -49.953067779541016,
+      "loss": 0.6114,
+      "losses/dpo": 0.6396130323410034,
+      "losses/sft": 1.41642427444458,
+      "losses/total": 0.6396130323410034,
+      "ref_logps/chosen": -36.14557647705078,
+      "ref_logps/rejected": -42.22886657714844,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.5588902235031128,
+      "rewards/margins": 0.21353021264076233,
+      "rewards/rejected": -0.7724204063415527,
+      "step": 137
+    },
+    {
+      "epoch": 1.04,
+      "grad_norm": 6.8531162605456055,
+      "learning_rate": 3.6235955056179776e-07,
+      "logps/chosen": -40.27118682861328,
+      "logps/rejected": -51.58380889892578,
+      "loss": 0.6205,
+      "losses/dpo": 0.6927103400230408,
+      "losses/sft": 1.5446867942810059,
+      "losses/total": 0.6927103400230408,
+      "ref_logps/chosen": -34.374351501464844,
+      "ref_logps/rejected": -43.53186798095703,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.5896837115287781,
+      "rewards/margins": 0.2155105173587799,
+      "rewards/rejected": -0.8051942586898804,
+      "step": 138
+    },
+    {
+      "epoch": 1.05,
+      "grad_norm": 6.581792409395151,
+      "learning_rate": 3.6095505617977526e-07,
+      "logps/chosen": -40.182411193847656,
+      "logps/rejected": -50.48395538330078,
+      "loss": 0.6351,
+      "losses/dpo": 0.5907813906669617,
+      "losses/sft": 1.5051367282867432,
+      "losses/total": 0.5907813906669617,
+      "ref_logps/chosen": -34.705772399902344,
+      "ref_logps/rejected": -43.20262908935547,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.5476638078689575,
+      "rewards/margins": 0.18046864867210388,
+      "rewards/rejected": -0.728132426738739,
+      "step": 139
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 7.099683859328159,
+      "learning_rate": 3.5955056179775277e-07,
+      "logps/chosen": -47.343101501464844,
+      "logps/rejected": -54.016380310058594,
+      "loss": 0.5962,
+      "losses/dpo": 0.5484156012535095,
+      "losses/sft": 1.3340450525283813,
+      "losses/total": 0.5484156012535095,
+      "ref_logps/chosen": -40.93060302734375,
+      "ref_logps/rejected": -45.07525634765625,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.6412495374679565,
+      "rewards/margins": 0.2528632581233978,
+      "rewards/rejected": -0.8941128253936768,
+      "step": 140
+    },
+    {
+      "epoch": 1.06,
+      "grad_norm": 6.6246858434595435,
+      "learning_rate": 3.581460674157303e-07,
+      "logps/chosen": -42.795745849609375,
+      "logps/rejected": -50.99128341674805,
+      "loss": 0.6105,
+      "losses/dpo": 0.6333677768707275,
+      "losses/sft": 1.7310549020767212,
+      "losses/total": 0.6333677768707275,
+      "ref_logps/chosen": -36.29172897338867,
+      "ref_logps/rejected": -42.22053527832031,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.6504020690917969,
+      "rewards/margins": 0.2266732007265091,
+      "rewards/rejected": -0.8770751953125,
+      "step": 141
+    },
+    {
+      "epoch": 1.07,
+      "grad_norm": 6.585988374877859,
+      "learning_rate": 3.5674157303370783e-07,
+      "logps/chosen": -39.767051696777344,
+      "logps/rejected": -45.62493896484375,
+      "loss": 0.6184,
+      "losses/dpo": 0.6107473373413086,
+      "losses/sft": 1.3934905529022217,
+      "losses/total": 0.6107473373413086,
+      "ref_logps/chosen": -34.30314636230469,
+      "ref_logps/rejected": -38.085365295410156,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.5463899374008179,
+      "rewards/margins": 0.20756718516349792,
+      "rewards/rejected": -0.7539570927619934,
+      "step": 142
+    },
+    {
+      "epoch": 1.08,
+      "grad_norm": 7.605347145021406,
+      "learning_rate": 3.553370786516854e-07,
+      "logps/chosen": -45.32689666748047,
+      "logps/rejected": -53.28538513183594,
+      "loss": 0.6396,
+      "losses/dpo": 0.6139785051345825,
+      "losses/sft": 1.4996390342712402,
+      "losses/total": 0.6139785051345825,
+      "ref_logps/chosen": -38.790870666503906,
+      "ref_logps/rejected": -44.86207580566406,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.6536027789115906,
+      "rewards/margins": 0.18872803449630737,
+      "rewards/rejected": -0.842330813407898,
+      "step": 143
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 6.375447848895096,
+      "learning_rate": 3.539325842696629e-07,
+      "logps/chosen": -39.950706481933594,
+      "logps/rejected": -52.67605972290039,
+      "loss": 0.6352,
+      "losses/dpo": 0.562360405921936,
+      "losses/sft": 1.5273569822311401,
+      "losses/total": 0.562360405921936,
+      "ref_logps/chosen": -33.4902458190918,
+      "ref_logps/rejected": -44.185813903808594,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.6460464000701904,
+      "rewards/margins": 0.20297789573669434,
+      "rewards/rejected": -0.8490242958068848,
+      "step": 144
+    },
+    {
+      "epoch": 1.09,
+      "grad_norm": 6.570913856555609,
+      "learning_rate": 3.5252808988764045e-07,
+      "logps/chosen": -41.82993698120117,
+      "logps/rejected": -49.303138732910156,
+      "loss": 0.616,
+      "losses/dpo": 0.6345305442810059,
+      "losses/sft": 1.6690033674240112,
+      "losses/total": 0.6345305442810059,
+      "ref_logps/chosen": -36.14568328857422,
+      "ref_logps/rejected": -41.49669647216797,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.5684253573417664,
+      "rewards/margins": 0.2122190296649933,
+      "rewards/rejected": -0.7806443572044373,
+      "step": 145
+    },
+    {
+      "epoch": 1.1,
+      "grad_norm": 6.513567190172644,
+      "learning_rate": 3.51123595505618e-07,
+      "logps/chosen": -41.52465057373047,
+      "logps/rejected": -53.12078094482422,
+      "loss": 0.6021,
+      "losses/dpo": 0.6336867809295654,
+      "losses/sft": 1.6118431091308594,
+      "losses/total": 0.6336867809295654,
+      "ref_logps/chosen": -35.433265686035156,
+      "ref_logps/rejected": -44.33394241333008,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.6091387271881104,
+      "rewards/margins": 0.2695454955101013,
+      "rewards/rejected": -0.8786842823028564,
+      "step": 146
+    },
+    {
+      "epoch": 1.11,
+      "grad_norm": 7.159126675968495,
+      "learning_rate": 3.497191011235955e-07,
+      "logps/chosen": -45.985076904296875,
+      "logps/rejected": -56.15589904785156,
+      "loss": 0.6053,
+      "losses/dpo": 0.6400711536407471,
+      "losses/sft": 1.618746042251587,
+      "losses/total": 0.6400711536407471,
+      "ref_logps/chosen": -39.94523239135742,
+      "ref_logps/rejected": -47.24679946899414,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.6039848327636719,
+      "rewards/margins": 0.2869252562522888,
+      "rewards/rejected": -0.8909100294113159,
+      "step": 147
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 6.31408614588678,
+      "learning_rate": 3.48314606741573e-07,
+      "logps/chosen": -36.47071075439453,
+      "logps/rejected": -44.49042510986328,
+      "loss": 0.6059,
+      "losses/dpo": 0.6014193892478943,
+      "losses/sft": 1.4461239576339722,
+      "losses/total": 0.6014193892478943,
+      "ref_logps/chosen": -31.124897003173828,
+      "ref_logps/rejected": -36.84246063232422,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.5345816016197205,
+      "rewards/margins": 0.23021462559700012,
+      "rewards/rejected": -0.7647961974143982,
+      "step": 148
+    },
+    {
+      "epoch": 1.12,
+      "grad_norm": 6.681653921230616,
+      "learning_rate": 3.469101123595505e-07,
+      "logps/chosen": -47.83122634887695,
+      "logps/rejected": -53.77499008178711,
+      "loss": 0.5731,
+      "losses/dpo": 0.5383450388908386,
+      "losses/sft": 1.5721720457077026,
+      "losses/total": 0.5383450388908386,
+      "ref_logps/chosen": -41.766536712646484,
+      "ref_logps/rejected": -44.522464752197266,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.6064690947532654,
+      "rewards/margins": 0.3187834620475769,
+      "rewards/rejected": -0.9252525568008423,
+      "step": 149
+    },
+    {
+      "epoch": 1.13,
+      "grad_norm": 6.904604135283266,
+      "learning_rate": 3.4550561797752807e-07,
+      "logps/chosen": -44.048912048339844,
+      "logps/rejected": -51.318206787109375,
+      "loss": 0.6352,
+      "losses/dpo": 0.7056742906570435,
+      "losses/sft": 1.706668496131897,
+      "losses/total": 0.7056742906570435,
+      "ref_logps/chosen": -37.56008529663086,
+      "ref_logps/rejected": -42.8933219909668,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.6488831043243408,
+      "rewards/margins": 0.19360551238059998,
+      "rewards/rejected": -0.8424886465072632,
+      "step": 150
+    },
+    {
+      "epoch": 1.14,
+      "grad_norm": 6.64022622762387,
+      "learning_rate": 3.441011235955056e-07,
+      "logps/chosen": -43.570884704589844,
+      "logps/rejected": -52.240318298339844,
+      "loss": 0.593,
+      "losses/dpo": 0.6028671264648438,
+      "losses/sft": 1.4086356163024902,
+      "losses/total": 0.6028671264648438,
+      "ref_logps/chosen": -38.0629768371582,
+      "ref_logps/rejected": -43.7314453125,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.5507906079292297,
+      "rewards/margins": 0.3000965714454651,
+      "rewards/rejected": -0.8508871793746948,
+      "step": 151
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 6.233323532005839,
+      "learning_rate": 3.4269662921348313e-07,
+      "logps/chosen": -37.21058654785156,
+      "logps/rejected": -48.0037841796875,
+      "loss": 0.594,
+      "losses/dpo": 0.614506721496582,
+      "losses/sft": 1.60889732837677,
+      "losses/total": 0.614506721496582,
+      "ref_logps/chosen": -31.845245361328125,
+      "ref_logps/rejected": -39.9993782043457,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.536533772945404,
+      "rewards/margins": 0.2639070749282837,
+      "rewards/rejected": -0.800440788269043,
+      "step": 152
+    },
+    {
+      "epoch": 1.15,
+      "grad_norm": 6.3687654295239335,
+      "learning_rate": 3.4129213483146064e-07,
+      "logps/chosen": -38.31336975097656,
+      "logps/rejected": -46.770484924316406,
+      "loss": 0.6353,
+      "losses/dpo": 0.67460036277771,
+      "losses/sft": 1.5722901821136475,
+      "losses/total": 0.67460036277771,
+      "ref_logps/chosen": -32.139488220214844,
+      "ref_logps/rejected": -38.692901611328125,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.617388129234314,
+      "rewards/margins": 0.1903703510761261,
+      "rewards/rejected": -0.8077584505081177,
+      "step": 153
+    },
+    {
+      "epoch": 1.16,
+      "grad_norm": 6.739574073940404,
+      "learning_rate": 3.398876404494382e-07,
+      "logps/chosen": -40.92176055908203,
+      "logps/rejected": -55.60083770751953,
+      "loss": 0.5768,
+      "losses/dpo": 0.6374070644378662,
+      "losses/sft": 1.6940921545028687,
+      "losses/total": 0.6374070644378662,
+      "ref_logps/chosen": -34.817481994628906,
+      "ref_logps/rejected": -46.21774673461914,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.6104279160499573,
+      "rewards/margins": 0.3278810977935791,
+      "rewards/rejected": -0.9383090138435364,
+      "step": 154
+    },
+    {
+      "epoch": 1.17,
+      "grad_norm": 6.879053261576145,
+      "learning_rate": 3.3848314606741575e-07,
+      "logps/chosen": -42.9251823425293,
+      "logps/rejected": -51.702964782714844,
+      "loss": 0.6139,
+      "losses/dpo": 0.6116975545883179,
+      "losses/sft": 1.40887451171875,
+      "losses/total": 0.6116975545883179,
+      "ref_logps/chosen": -37.03919982910156,
+      "ref_logps/rejected": -43.293392181396484,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.5885984301567078,
+      "rewards/margins": 0.25235864520072937,
+      "rewards/rejected": -0.8409571051597595,
+      "step": 155
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 6.694865071581937,
+      "learning_rate": 3.3707865168539325e-07,
+      "logps/chosen": -42.003990173339844,
+      "logps/rejected": -50.280548095703125,
+      "loss": 0.6314,
+      "losses/dpo": 0.6589547991752625,
+      "losses/sft": 1.475001573562622,
+      "losses/total": 0.6589547991752625,
+      "ref_logps/chosen": -35.952449798583984,
+      "ref_logps/rejected": -42.320465087890625,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.6051540374755859,
+      "rewards/margins": 0.1908540576696396,
+      "rewards/rejected": -0.7960080504417419,
+      "step": 156
+    },
+    {
+      "epoch": 1.18,
+      "grad_norm": 6.680369593277332,
+      "learning_rate": 3.356741573033708e-07,
+      "logps/chosen": -38.10074996948242,
+      "logps/rejected": -47.0694694519043,
+      "loss": 0.6165,
+      "losses/dpo": 0.5133580565452576,
+      "losses/sft": 1.357291340827942,
+      "losses/total": 0.5133580565452576,
+      "ref_logps/chosen": -32.30625915527344,
+      "ref_logps/rejected": -38.88528060913086,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.5794489979743958,
+      "rewards/margins": 0.23896978795528412,
+      "rewards/rejected": -0.8184187412261963,
+      "step": 157
+    },
+    {
+      "epoch": 1.19,
+      "grad_norm": 7.163827933877828,
+      "learning_rate": 3.3426966292134826e-07,
+      "logps/chosen": -43.57667541503906,
+      "logps/rejected": -53.30217742919922,
+      "loss": 0.5908,
+      "losses/dpo": 0.574418306350708,
+      "losses/sft": 1.7125813961029053,
+      "losses/total": 0.574418306350708,
+      "ref_logps/chosen": -37.39282989501953,
+      "ref_logps/rejected": -44.2191162109375,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.6183844804763794,
+      "rewards/margins": 0.2899210453033447,
+      "rewards/rejected": -0.9083055257797241,
+      "step": 158
+    },
+    {
+      "epoch": 1.2,
+      "grad_norm": 7.534492030316663,
+      "learning_rate": 3.328651685393258e-07,
+      "logps/chosen": -43.87846755981445,
+      "logps/rejected": -50.815635681152344,
+      "loss": 0.6314,
+      "losses/dpo": 0.6066723465919495,
+      "losses/sft": 1.625878930091858,
+      "losses/total": 0.6066723465919495,
+      "ref_logps/chosen": -37.40098190307617,
+      "ref_logps/rejected": -42.1273193359375,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.6477489471435547,
+      "rewards/margins": 0.2210829257965088,
+      "rewards/rejected": -0.8688318729400635,
+      "step": 159
+    },
+    {
+      "epoch": 1.21,
+      "grad_norm": 6.461969936591063,
+      "learning_rate": 3.314606741573033e-07,
+      "logps/chosen": -41.52513122558594,
+      "logps/rejected": -50.716064453125,
+      "loss": 0.5689,
+      "losses/dpo": 0.5582201480865479,
+      "losses/sft": 1.5073952674865723,
+      "losses/total": 0.5582201480865479,
+      "ref_logps/chosen": -35.848846435546875,
+      "ref_logps/rejected": -41.70889663696289,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.567628026008606,
+      "rewards/margins": 0.333088219165802,
+      "rewards/rejected": -0.9007163047790527,
+      "step": 160
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 6.942966081986398,
+      "learning_rate": 3.300561797752809e-07,
+      "logps/chosen": -45.35464859008789,
+      "logps/rejected": -48.982810974121094,
+      "loss": 0.6365,
+      "losses/dpo": 0.5614318251609802,
+      "losses/sft": 1.7859472036361694,
+      "losses/total": 0.5614318251609802,
+      "ref_logps/chosen": -39.001136779785156,
+      "ref_logps/rejected": -40.674232482910156,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.6353514790534973,
+      "rewards/margins": 0.19550636410713196,
+      "rewards/rejected": -0.8308578729629517,
+      "step": 161
+    },
+    {
+      "epoch": 1.22,
+      "grad_norm": 7.180092377812206,
+      "learning_rate": 3.2865168539325844e-07,
+      "logps/chosen": -45.20994186401367,
+      "logps/rejected": -52.836875915527344,
+      "loss": 0.6224,
+      "losses/dpo": 0.5493739247322083,
+      "losses/sft": 1.547910213470459,
+      "losses/total": 0.5493739247322083,
+      "ref_logps/chosen": -38.36485290527344,
+      "ref_logps/rejected": -43.65592575073242,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.684508740901947,
+      "rewards/margins": 0.2335864156484604,
+      "rewards/rejected": -0.9180951714515686,
+      "step": 162
+    },
+    {
+      "epoch": 1.23,
+      "grad_norm": 6.839328465422381,
+      "learning_rate": 3.2724719101123594e-07,
+      "logps/chosen": -44.02252197265625,
+      "logps/rejected": -48.20290756225586,
+      "loss": 0.6261,
+      "losses/dpo": 0.5617036819458008,
+      "losses/sft": 1.5726804733276367,
+      "losses/total": 0.5617036819458008,
+      "ref_logps/chosen": -37.56252670288086,
+      "ref_logps/rejected": -39.5811882019043,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.6459991931915283,
+      "rewards/margins": 0.21617242693901062,
+      "rewards/rejected": -0.8621717095375061,
+      "step": 163
+    },
+    {
+      "epoch": 1.24,
+      "grad_norm": 7.080895585848076,
+      "learning_rate": 3.258426966292135e-07,
+      "logps/chosen": -44.87802505493164,
+      "logps/rejected": -53.4232292175293,
+      "loss": 0.6444,
+      "losses/dpo": 0.5438011884689331,
+      "losses/sft": 1.6607606410980225,
+      "losses/total": 0.5438011884689331,
+      "ref_logps/chosen": -37.73242950439453,
+      "ref_logps/rejected": -44.338134765625,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.7145596146583557,
+      "rewards/margins": 0.1939493715763092,
+      "rewards/rejected": -0.9085089564323425,
+      "step": 164
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 6.817784306171125,
+      "learning_rate": 3.24438202247191e-07,
+      "logps/chosen": -41.74745559692383,
+      "logps/rejected": -54.77409362792969,
+      "loss": 0.5757,
+      "losses/dpo": 0.5074477195739746,
+      "losses/sft": 1.4384926557540894,
+      "losses/total": 0.5074477195739746,
+      "ref_logps/chosen": -35.96852493286133,
+      "ref_logps/rejected": -45.66744613647461,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.5778931379318237,
+      "rewards/margins": 0.3327715992927551,
+      "rewards/rejected": -0.9106647372245789,
+      "step": 165
+    },
+    {
+      "epoch": 1.25,
+      "grad_norm": 6.73402019836184,
+      "learning_rate": 3.2303370786516856e-07,
+      "logps/chosen": -42.45885467529297,
+      "logps/rejected": -55.83628845214844,
+      "loss": 0.5582,
+      "losses/dpo": 0.5949134230613708,
+      "losses/sft": 1.7396336793899536,
+      "losses/total": 0.5949134230613708,
+      "ref_logps/chosen": -35.91239547729492,
+      "ref_logps/rejected": -45.41798400878906,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.6546458601951599,
+      "rewards/margins": 0.387184739112854,
+      "rewards/rejected": -1.0418306589126587,
+      "step": 166
+    },
+    {
+      "epoch": 1.26,
+      "grad_norm": 6.484969223325532,
+      "learning_rate": 3.21629213483146e-07,
+      "logps/chosen": -39.55500030517578,
+      "logps/rejected": -53.75917053222656,
+      "loss": 0.5553,
+      "losses/dpo": 0.5379188060760498,
+      "losses/sft": 1.6754546165466309,
+      "losses/total": 0.5379188060760498,
+      "ref_logps/chosen": -33.612388610839844,
+      "ref_logps/rejected": -44.048587799072266,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.5942604541778564,
+      "rewards/margins": 0.3767976760864258,
+      "rewards/rejected": -0.971058189868927,
+      "step": 167
+    },
+    {
+      "epoch": 1.27,
+      "grad_norm": 7.553429925940112,
+      "learning_rate": 3.2022471910112357e-07,
+      "logps/chosen": -42.48908615112305,
+      "logps/rejected": -53.703208923339844,
+      "loss": 0.6156,
+      "losses/dpo": 0.5968649983406067,
+      "losses/sft": 1.4021852016448975,
+      "losses/total": 0.5968649983406067,
+      "ref_logps/chosen": -36.02775573730469,
+      "ref_logps/rejected": -44.81937026977539,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.6461330056190491,
+      "rewards/margins": 0.24225082993507385,
+      "rewards/rejected": -0.8883838057518005,
+      "step": 168
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 6.938904519141351,
+      "learning_rate": 3.1882022471910107e-07,
+      "logps/chosen": -41.907588958740234,
+      "logps/rejected": -51.34623718261719,
+      "loss": 0.606,
+      "losses/dpo": 0.5469992756843567,
+      "losses/sft": 1.526263952255249,
+      "losses/total": 0.5469992756843567,
+      "ref_logps/chosen": -35.054046630859375,
+      "ref_logps/rejected": -41.81439971923828,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.6853541135787964,
+      "rewards/margins": 0.26782965660095215,
+      "rewards/rejected": -0.9531837701797485,
+      "step": 169
+    },
+    {
+      "epoch": 1.28,
+      "grad_norm": 6.455751734226468,
+      "learning_rate": 3.1741573033707863e-07,
+      "logps/chosen": -43.997074127197266,
+      "logps/rejected": -52.1095085144043,
+      "loss": 0.5652,
+      "losses/dpo": 0.5796064138412476,
+      "losses/sft": 1.7088840007781982,
+      "losses/total": 0.5796064138412476,
+      "ref_logps/chosen": -37.47534942626953,
+      "ref_logps/rejected": -42.05298614501953,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.6521726846694946,
+      "rewards/margins": 0.35347938537597656,
+      "rewards/rejected": -1.0056521892547607,
+      "step": 170
+    },
+    {
+      "epoch": 1.29,
+      "grad_norm": 6.925720662101029,
+      "learning_rate": 3.160112359550562e-07,
+      "logps/chosen": -42.9494743347168,
+      "logps/rejected": -51.66035461425781,
+      "loss": 0.5733,
+      "losses/dpo": 0.5771209001541138,
+      "losses/sft": 1.3783965110778809,
+      "losses/total": 0.5771209001541138,
+      "ref_logps/chosen": -36.78116989135742,
+      "ref_logps/rejected": -41.843605041503906,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.6168303489685059,
+      "rewards/margins": 0.36484503746032715,
+      "rewards/rejected": -0.981675386428833,
+      "step": 171
+    },
+    {
+      "epoch": 1.3,
+      "grad_norm": 7.01889024226675,
+      "learning_rate": 3.146067415730337e-07,
+      "logps/chosen": -45.94253158569336,
+      "logps/rejected": -52.16916275024414,
+      "loss": 0.6015,
+      "losses/dpo": 0.5898208618164062,
+      "losses/sft": 1.6218047142028809,
+      "losses/total": 0.5898208618164062,
+      "ref_logps/chosen": -39.25119400024414,
+      "ref_logps/rejected": -42.79926300048828,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.669133722782135,
+      "rewards/margins": 0.2678561806678772,
+      "rewards/rejected": -0.936989963054657,
+      "step": 172
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 7.495610381062607,
+      "learning_rate": 3.1320224719101125e-07,
+      "logps/chosen": -44.17959976196289,
+      "logps/rejected": -53.05359649658203,
+      "loss": 0.6002,
+      "losses/dpo": 0.630514919757843,
+      "losses/sft": 1.7863552570343018,
+      "losses/total": 0.630514919757843,
+      "ref_logps/chosen": -37.75782012939453,
+      "ref_logps/rejected": -43.63001251220703,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.642177939414978,
+      "rewards/margins": 0.30018070340156555,
+      "rewards/rejected": -0.9423586130142212,
+      "step": 173
+    },
+    {
+      "epoch": 1.31,
+      "grad_norm": 7.4434324895050015,
+      "learning_rate": 3.1179775280898875e-07,
+      "logps/chosen": -47.16429901123047,
+      "logps/rejected": -53.266780853271484,
+      "loss": 0.6233,
+      "losses/dpo": 0.5390438437461853,
+      "losses/sft": 1.480837345123291,
+      "losses/total": 0.5390438437461853,
+      "ref_logps/chosen": -40.324798583984375,
+      "ref_logps/rejected": -44.025787353515625,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.6839500069618225,
+      "rewards/margins": 0.24014970660209656,
+      "rewards/rejected": -0.9240997433662415,
+      "step": 174
+    },
+    {
+      "epoch": 1.32,
+      "grad_norm": 7.0617610747390644,
+      "learning_rate": 3.103932584269663e-07,
+      "logps/chosen": -45.060882568359375,
+      "logps/rejected": -54.979156494140625,
+      "loss": 0.5891,
+      "losses/dpo": 0.5568109750747681,
+      "losses/sft": 1.6295528411865234,
+      "losses/total": 0.5568109750747681,
+      "ref_logps/chosen": -37.929840087890625,
+      "ref_logps/rejected": -44.73991394042969,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7131036520004272,
+      "rewards/margins": 0.3108205795288086,
+      "rewards/rejected": -1.0239241123199463,
+      "step": 175
+    },
+    {
+      "epoch": 1.33,
+      "grad_norm": 6.663174112724157,
+      "learning_rate": 3.0898876404494376e-07,
+      "logps/chosen": -42.657920837402344,
+      "logps/rejected": -47.67673110961914,
+      "loss": 0.629,
+      "losses/dpo": 0.7223004102706909,
+      "losses/sft": 1.3237264156341553,
+      "losses/total": 0.7223004102706909,
+      "ref_logps/chosen": -36.358001708984375,
+      "ref_logps/rejected": -38.938507080078125,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.6299920082092285,
+      "rewards/margins": 0.24383032321929932,
+      "rewards/rejected": -0.8738222122192383,
+      "step": 176
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 7.940936921413792,
+      "learning_rate": 3.075842696629213e-07,
+      "logps/chosen": -48.067203521728516,
+      "logps/rejected": -52.38322067260742,
+      "loss": 0.6277,
+      "losses/dpo": 0.6528229117393494,
+      "losses/sft": 1.5663461685180664,
+      "losses/total": 0.6528229117393494,
+      "ref_logps/chosen": -40.93121337890625,
+      "ref_logps/rejected": -42.879188537597656,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7135992050170898,
+      "rewards/margins": 0.23680387437343597,
+      "rewards/rejected": -0.950403094291687,
+      "step": 177
+    },
+    {
+      "epoch": 1.34,
+      "grad_norm": 6.9909555671219366,
+      "learning_rate": 3.0617977528089887e-07,
+      "logps/chosen": -43.74810791015625,
+      "logps/rejected": -51.602622985839844,
+      "loss": 0.6134,
+      "losses/dpo": 0.6351133584976196,
+      "losses/sft": 1.548452615737915,
+      "losses/total": 0.6351133584976196,
+      "ref_logps/chosen": -36.539649963378906,
+      "ref_logps/rejected": -41.724647521972656,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.7208462953567505,
+      "rewards/margins": 0.26695096492767334,
+      "rewards/rejected": -0.9877973794937134,
+      "step": 178
+    },
+    {
+      "epoch": 1.35,
+      "grad_norm": 7.516487264439432,
+      "learning_rate": 3.047752808988764e-07,
+      "logps/chosen": -45.41664123535156,
+      "logps/rejected": -52.19443130493164,
+      "loss": 0.6325,
+      "losses/dpo": 0.6936246752738953,
+      "losses/sft": 1.418731451034546,
+      "losses/total": 0.6936246752738953,
+      "ref_logps/chosen": -37.89662170410156,
+      "ref_logps/rejected": -42.46479797363281,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.7520017623901367,
+      "rewards/margins": 0.22096163034439087,
+      "rewards/rejected": -0.9729634523391724,
+      "step": 179
+    },
+    {
+      "epoch": 1.36,
+      "grad_norm": 7.386682971240318,
+      "learning_rate": 3.0337078651685393e-07,
+      "logps/chosen": -44.32128143310547,
+      "logps/rejected": -56.52943420410156,
+      "loss": 0.589,
+      "losses/dpo": 0.50272536277771,
+      "losses/sft": 1.4175364971160889,
+      "losses/total": 0.50272536277771,
+      "ref_logps/chosen": -36.94242477416992,
+      "ref_logps/rejected": -45.8451042175293,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.7378860712051392,
+      "rewards/margins": 0.3305472731590271,
+      "rewards/rejected": -1.0684332847595215,
+      "step": 180
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 7.070520625555024,
+      "learning_rate": 3.0196629213483144e-07,
+      "logps/chosen": -42.875099182128906,
+      "logps/rejected": -52.050384521484375,
+      "loss": 0.5815,
+      "losses/dpo": 0.6107900738716125,
+      "losses/sft": 1.7456879615783691,
+      "losses/total": 0.6107900738716125,
+      "ref_logps/chosen": -36.82639694213867,
+      "ref_logps/rejected": -42.541603088378906,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.604870617389679,
+      "rewards/margins": 0.34600716829299927,
+      "rewards/rejected": -0.9508777856826782,
+      "step": 181
+    },
+    {
+      "epoch": 1.37,
+      "grad_norm": 6.880606354027993,
+      "learning_rate": 3.00561797752809e-07,
+      "logps/chosen": -44.50994110107422,
+      "logps/rejected": -50.33254623413086,
+      "loss": 0.5976,
+      "losses/dpo": 0.5528784990310669,
+      "losses/sft": 1.7669578790664673,
+      "losses/total": 0.5528784990310669,
+      "ref_logps/chosen": -37.44862365722656,
+      "ref_logps/rejected": -40.247039794921875,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.7061322331428528,
+      "rewards/margins": 0.30241847038269043,
+      "rewards/rejected": -1.0085506439208984,
+      "step": 182
+    },
+    {
+      "epoch": 1.38,
+      "grad_norm": 7.02791538850361,
+      "learning_rate": 2.991573033707865e-07,
+      "logps/chosen": -43.87753677368164,
+      "logps/rejected": -49.34949493408203,
+      "loss": 0.6184,
+      "losses/dpo": 0.5282893180847168,
+      "losses/sft": 1.5792714357376099,
+      "losses/total": 0.5282893180847168,
+      "ref_logps/chosen": -36.59724044799805,
+      "ref_logps/rejected": -39.5660285949707,
+      "rewards/accuracies": 0.59375,
+      "rewards/chosen": -0.7280292510986328,
+      "rewards/margins": 0.25031745433807373,
+      "rewards/rejected": -0.9783467054367065,
+      "step": 183
+    },
+    {
+      "epoch": 1.39,
+      "grad_norm": 7.145778710825677,
+      "learning_rate": 2.9775280898876406e-07,
+      "logps/chosen": -44.12153625488281,
+      "logps/rejected": -50.03323745727539,
+      "loss": 0.5994,
+      "losses/dpo": 0.5445400476455688,
+      "losses/sft": 1.3715200424194336,
+      "losses/total": 0.5445400476455688,
+      "ref_logps/chosen": -36.779212951660156,
+      "ref_logps/rejected": -39.96929168701172,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7342325448989868,
+      "rewards/margins": 0.2721615731716156,
+      "rewards/rejected": -1.0063941478729248,
+      "step": 184
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 7.033244427636761,
+      "learning_rate": 2.9634831460674156e-07,
+      "logps/chosen": -44.52336120605469,
+      "logps/rejected": -52.357810974121094,
+      "loss": 0.5954,
+      "losses/dpo": 0.7007085084915161,
+      "losses/sft": 1.6124120950698853,
+      "losses/total": 0.7007085084915161,
+      "ref_logps/chosen": -37.63973617553711,
+      "ref_logps/rejected": -42.18433380126953,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6883625984191895,
+      "rewards/margins": 0.32898518443107605,
+      "rewards/rejected": -1.017347812652588,
+      "step": 185
+    },
+    {
+      "epoch": 1.4,
+      "grad_norm": 6.657232520968286,
+      "learning_rate": 2.9494382022471906e-07,
+      "logps/chosen": -41.616920471191406,
+      "logps/rejected": -52.240867614746094,
+      "loss": 0.5708,
+      "losses/dpo": 0.6222548484802246,
+      "losses/sft": 1.4851452112197876,
+      "losses/total": 0.6222548484802246,
+      "ref_logps/chosen": -35.22339630126953,
+      "ref_logps/rejected": -42.063621520996094,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.6393523216247559,
+      "rewards/margins": 0.37837234139442444,
+      "rewards/rejected": -1.0177247524261475,
+      "step": 186
+    },
+    {
+      "epoch": 1.41,
+      "grad_norm": 7.293022584639844,
+      "learning_rate": 2.935393258426966e-07,
+      "logps/chosen": -44.5795783996582,
+      "logps/rejected": -51.8583869934082,
+      "loss": 0.613,
+      "losses/dpo": 0.5626444816589355,
+      "losses/sft": 1.5801838636398315,
+      "losses/total": 0.5626444816589355,
+      "ref_logps/chosen": -37.548030853271484,
+      "ref_logps/rejected": -41.92090606689453,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.7031550407409668,
+      "rewards/margins": 0.2905934154987335,
+      "rewards/rejected": -0.9937484264373779,
+      "step": 187
+    },
+    {
+      "epoch": 1.42,
+      "grad_norm": 6.7646034429768385,
+      "learning_rate": 2.921348314606741e-07,
+      "logps/chosen": -38.493064880371094,
+      "logps/rejected": -52.74738693237305,
+      "loss": 0.5673,
+      "losses/dpo": 0.5454456210136414,
+      "losses/sft": 1.4984629154205322,
+      "losses/total": 0.5454456210136414,
+      "ref_logps/chosen": -31.663148880004883,
+      "ref_logps/rejected": -42.071197509765625,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.6829913854598999,
+      "rewards/margins": 0.38462772965431213,
+      "rewards/rejected": -1.0676190853118896,
+      "step": 188
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 6.461217065826113,
+      "learning_rate": 2.907303370786517e-07,
+      "logps/chosen": -41.286556243896484,
+      "logps/rejected": -50.088348388671875,
+      "loss": 0.5702,
+      "losses/dpo": 0.5214348435401917,
+      "losses/sft": 1.4683022499084473,
+      "losses/total": 0.5214348435401917,
+      "ref_logps/chosen": -35.07402038574219,
+      "ref_logps/rejected": -40.096168518066406,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.6212539672851562,
+      "rewards/margins": 0.377963662147522,
+      "rewards/rejected": -0.9992176294326782,
+      "step": 189
+    },
+    {
+      "epoch": 1.43,
+      "grad_norm": 7.005927567034465,
+      "learning_rate": 2.893258426966292e-07,
+      "logps/chosen": -40.85224533081055,
+      "logps/rejected": -47.50454330444336,
+      "loss": 0.6468,
+      "losses/dpo": 0.5624043941497803,
+      "losses/sft": 1.3726718425750732,
+      "losses/total": 0.5624043941497803,
+      "ref_logps/chosen": -34.436492919921875,
+      "ref_logps/rejected": -39.054630279541016,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.6415754556655884,
+      "rewards/margins": 0.2034158706665039,
+      "rewards/rejected": -0.8449913263320923,
+      "step": 190
+    },
+    {
+      "epoch": 1.44,
+      "grad_norm": 7.483874035437063,
+      "learning_rate": 2.8792134831460674e-07,
+      "logps/chosen": -42.50736999511719,
+      "logps/rejected": -58.21019744873047,
+      "loss": 0.5523,
+      "losses/dpo": 0.6698145270347595,
+      "losses/sft": 1.5408368110656738,
+      "losses/total": 0.6698145270347595,
+      "ref_logps/chosen": -36.81276321411133,
+      "ref_logps/rejected": -48.2528076171875,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.5694608688354492,
+      "rewards/margins": 0.4262778162956238,
+      "rewards/rejected": -0.995738685131073,
+      "step": 191
+    },
+    {
+      "epoch": 1.45,
+      "grad_norm": 6.9825083587199,
+      "learning_rate": 2.8651685393258425e-07,
+      "logps/chosen": -45.57709503173828,
+      "logps/rejected": -53.14373016357422,
+      "loss": 0.5855,
+      "losses/dpo": 0.557357132434845,
+      "losses/sft": 1.6354026794433594,
+      "losses/total": 0.557357132434845,
+      "ref_logps/chosen": -39.346527099609375,
+      "ref_logps/rejected": -43.652854919433594,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.6230565309524536,
+      "rewards/margins": 0.32603132724761963,
+      "rewards/rejected": -0.9490878582000732,
+      "step": 192
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 6.9248793596215314,
+      "learning_rate": 2.851123595505618e-07,
+      "logps/chosen": -41.4918098449707,
+      "logps/rejected": -51.642330169677734,
+      "loss": 0.614,
+      "losses/dpo": 0.5539823770523071,
+      "losses/sft": 1.4280143976211548,
+      "losses/total": 0.5539823770523071,
+      "ref_logps/chosen": -34.640289306640625,
+      "ref_logps/rejected": -42.01956558227539,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.6851522326469421,
+      "rewards/margins": 0.2771243751049042,
+      "rewards/rejected": -0.9622765779495239,
+      "step": 193
+    },
+    {
+      "epoch": 1.46,
+      "grad_norm": 7.819580369685109,
+      "learning_rate": 2.8370786516853936e-07,
+      "logps/chosen": -45.17947769165039,
+      "logps/rejected": -54.26673126220703,
+      "loss": 0.5983,
+      "losses/dpo": 0.5532131195068359,
+      "losses/sft": 1.5786592960357666,
+      "losses/total": 0.5532131195068359,
+      "ref_logps/chosen": -38.547950744628906,
+      "ref_logps/rejected": -44.515296936035156,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6631526350975037,
+      "rewards/margins": 0.3119913339614868,
+      "rewards/rejected": -0.9751439094543457,
+      "step": 194
+    },
+    {
+      "epoch": 1.47,
+      "grad_norm": 6.908089828532824,
+      "learning_rate": 2.823033707865168e-07,
+      "logps/chosen": -39.231468200683594,
+      "logps/rejected": -55.21925735473633,
+      "loss": 0.5648,
+      "losses/dpo": 0.5973429083824158,
+      "losses/sft": 1.6660652160644531,
+      "losses/total": 0.5973429083824158,
+      "ref_logps/chosen": -32.866737365722656,
+      "ref_logps/rejected": -44.704856872558594,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.6364729404449463,
+      "rewards/margins": 0.4149664640426636,
+      "rewards/rejected": -1.0514394044876099,
+      "step": 195
+    },
+    {
+      "epoch": 1.48,
+      "grad_norm": 6.726944591334497,
+      "learning_rate": 2.8089887640449437e-07,
+      "logps/chosen": -40.06050109863281,
+      "logps/rejected": -53.288673400878906,
+      "loss": 0.5791,
+      "losses/dpo": 0.5540711879730225,
+      "losses/sft": 1.7805967330932617,
+      "losses/total": 0.5540711879730225,
+      "ref_logps/chosen": -33.56330490112305,
+      "ref_logps/rejected": -42.997859954833984,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.649719774723053,
+      "rewards/margins": 0.37936151027679443,
+      "rewards/rejected": -1.0290813446044922,
+      "step": 196
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 7.150851904029176,
+      "learning_rate": 2.794943820224719e-07,
+      "logps/chosen": -47.1893424987793,
+      "logps/rejected": -61.44281005859375,
+      "loss": 0.5702,
+      "losses/dpo": 0.7234626412391663,
+      "losses/sft": 1.6843864917755127,
+      "losses/total": 0.7234626412391663,
+      "ref_logps/chosen": -39.28502655029297,
+      "ref_logps/rejected": -49.627784729003906,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7904319763183594,
+      "rewards/margins": 0.39107024669647217,
+      "rewards/rejected": -1.1815022230148315,
+      "step": 197
+    },
+    {
+      "epoch": 1.49,
+      "grad_norm": 7.427853846385361,
+      "learning_rate": 2.7808988764044943e-07,
+      "logps/chosen": -43.90837097167969,
+      "logps/rejected": -49.889678955078125,
+      "loss": 0.6097,
+      "losses/dpo": 0.5974606275558472,
+      "losses/sft": 1.7023361921310425,
+      "losses/total": 0.5974606275558472,
+      "ref_logps/chosen": -36.68721008300781,
+      "ref_logps/rejected": -39.80302047729492,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.722116231918335,
+      "rewards/margins": 0.2865493595600128,
+      "rewards/rejected": -1.0086655616760254,
+      "step": 198
+    },
+    {
+      "epoch": 1.5,
+      "grad_norm": 6.720052652716852,
+      "learning_rate": 2.7668539325842694e-07,
+      "logps/chosen": -40.47029495239258,
+      "logps/rejected": -52.58824157714844,
+      "loss": 0.5673,
+      "losses/dpo": 0.5275993347167969,
+      "losses/sft": 1.4116981029510498,
+      "losses/total": 0.5275993347167969,
+      "ref_logps/chosen": -33.68723678588867,
+      "ref_logps/rejected": -42.008827209472656,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6783058643341064,
+      "rewards/margins": 0.3796355426311493,
+      "rewards/rejected": -1.0579413175582886,
+      "step": 199
+    },
+    {
+      "epoch": 1.51,
+      "grad_norm": 7.464608685292226,
+      "learning_rate": 2.752808988764045e-07,
+      "logps/chosen": -46.24801254272461,
+      "logps/rejected": -54.933780670166016,
+      "loss": 0.61,
+      "losses/dpo": 0.6066948771476746,
+      "losses/sft": 1.6309008598327637,
+      "losses/total": 0.6066948771476746,
+      "ref_logps/chosen": -38.81559753417969,
+      "ref_logps/rejected": -44.58855438232422,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.743241548538208,
+      "rewards/margins": 0.2912812829017639,
+      "rewards/rejected": -1.0345228910446167,
+      "step": 200
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 7.360337757301619,
+      "learning_rate": 2.73876404494382e-07,
+      "logps/chosen": -42.876792907714844,
+      "logps/rejected": -50.461334228515625,
+      "loss": 0.6213,
+      "losses/dpo": 0.6310982704162598,
+      "losses/sft": 1.441427230834961,
+      "losses/total": 0.6310982704162598,
+      "ref_logps/chosen": -36.28274917602539,
+      "ref_logps/rejected": -41.36058044433594,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.6594043970108032,
+      "rewards/margins": 0.2506704330444336,
+      "rewards/rejected": -0.9100748300552368,
+      "step": 201
+    },
+    {
+      "epoch": 1.52,
+      "grad_norm": 7.43302729298079,
+      "learning_rate": 2.7247191011235955e-07,
+      "logps/chosen": -43.45911407470703,
+      "logps/rejected": -50.20298385620117,
+      "loss": 0.5552,
+      "losses/dpo": 0.5599596500396729,
+      "losses/sft": 1.4739470481872559,
+      "losses/total": 0.5599596500396729,
+      "ref_logps/chosen": -37.075191497802734,
+      "ref_logps/rejected": -39.420005798339844,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.6383919715881348,
+      "rewards/margins": 0.4399053752422333,
+      "rewards/rejected": -1.0782973766326904,
+      "step": 202
+    },
+    {
+      "epoch": 1.53,
+      "grad_norm": 7.05544065339559,
+      "learning_rate": 2.710674157303371e-07,
+      "logps/chosen": -48.13520050048828,
+      "logps/rejected": -55.488975524902344,
+      "loss": 0.5683,
+      "losses/dpo": 0.5512528419494629,
+      "losses/sft": 1.421828269958496,
+      "losses/total": 0.5512528419494629,
+      "ref_logps/chosen": -40.806800842285156,
+      "ref_logps/rejected": -44.204917907714844,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7328400611877441,
+      "rewards/margins": 0.39556559920310974,
+      "rewards/rejected": -1.1284055709838867,
+      "step": 203
+    },
+    {
+      "epoch": 1.54,
+      "grad_norm": 7.072397811821575,
+      "learning_rate": 2.6966292134831456e-07,
+      "logps/chosen": -45.89094161987305,
+      "logps/rejected": -56.12247085571289,
+      "loss": 0.5687,
+      "losses/dpo": 0.5942108035087585,
+      "losses/sft": 1.671670913696289,
+      "losses/total": 0.5942108035087585,
+      "ref_logps/chosen": -38.66583251953125,
+      "ref_logps/rejected": -45.235042572021484,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.7225111722946167,
+      "rewards/margins": 0.3662317395210266,
+      "rewards/rejected": -1.088742971420288,
+      "step": 204
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 7.135519070946246,
+      "learning_rate": 2.682584269662921e-07,
+      "logps/chosen": -44.51463317871094,
+      "logps/rejected": -53.46598434448242,
+      "loss": 0.5668,
+      "losses/dpo": 0.5319070816040039,
+      "losses/sft": 1.5628294944763184,
+      "losses/total": 0.5319070816040039,
+      "ref_logps/chosen": -37.279029846191406,
+      "ref_logps/rejected": -42.57787322998047,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7235599756240845,
+      "rewards/margins": 0.36525097489356995,
+      "rewards/rejected": -1.0888110399246216,
+      "step": 205
+    },
+    {
+      "epoch": 1.55,
+      "grad_norm": 7.107636740157782,
+      "learning_rate": 2.668539325842696e-07,
+      "logps/chosen": -43.40117645263672,
+      "logps/rejected": -54.69598388671875,
+      "loss": 0.5524,
+      "losses/dpo": 0.5264509320259094,
+      "losses/sft": 1.5363452434539795,
+      "losses/total": 0.5264509320259094,
+      "ref_logps/chosen": -36.371551513671875,
+      "ref_logps/rejected": -43.2625732421875,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.7029624581336975,
+      "rewards/margins": 0.44037845730781555,
+      "rewards/rejected": -1.1433409452438354,
+      "step": 206
+    },
+    {
+      "epoch": 1.56,
+      "grad_norm": 7.289414925759057,
+      "learning_rate": 2.654494382022472e-07,
+      "logps/chosen": -42.475379943847656,
+      "logps/rejected": -49.646728515625,
+      "loss": 0.6046,
+      "losses/dpo": 0.6572248935699463,
+      "losses/sft": 1.6387099027633667,
+      "losses/total": 0.6572248935699463,
+      "ref_logps/chosen": -35.308807373046875,
+      "ref_logps/rejected": -39.25323486328125,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.7166574597358704,
+      "rewards/margins": 0.32269221544265747,
+      "rewards/rejected": -1.0393496751785278,
+      "step": 207
+    },
+    {
+      "epoch": 1.57,
+      "grad_norm": 7.273876435078599,
+      "learning_rate": 2.640449438202247e-07,
+      "logps/chosen": -42.280967712402344,
+      "logps/rejected": -48.80766296386719,
+      "loss": 0.6213,
+      "losses/dpo": 0.5971169471740723,
+      "losses/sft": 1.7042605876922607,
+      "losses/total": 0.5971169471740723,
+      "ref_logps/chosen": -35.188377380371094,
+      "ref_logps/rejected": -39.048980712890625,
+      "rewards/accuracies": 0.6015625,
+      "rewards/chosen": -0.7092592716217041,
+      "rewards/margins": 0.2666093707084656,
+      "rewards/rejected": -0.9758686423301697,
+      "step": 208
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 7.389043621661051,
+      "learning_rate": 2.6264044943820224e-07,
+      "logps/chosen": -43.01720428466797,
+      "logps/rejected": -52.86360549926758,
+      "loss": 0.6004,
+      "losses/dpo": 0.5311284065246582,
+      "losses/sft": 1.673902988433838,
+      "losses/total": 0.5311284065246582,
+      "ref_logps/chosen": -35.34561538696289,
+      "ref_logps/rejected": -41.90196228027344,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.7671589851379395,
+      "rewards/margins": 0.3290054500102997,
+      "rewards/rejected": -1.096164345741272,
+      "step": 209
+    },
+    {
+      "epoch": 1.58,
+      "grad_norm": 7.051856361062949,
+      "learning_rate": 2.612359550561798e-07,
+      "logps/chosen": -43.410194396972656,
+      "logps/rejected": -56.95100784301758,
+      "loss": 0.5527,
+      "losses/dpo": 0.494179904460907,
+      "losses/sft": 1.3610440492630005,
+      "losses/total": 0.494179904460907,
+      "ref_logps/chosen": -36.352073669433594,
+      "ref_logps/rejected": -45.52418518066406,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.7058122754096985,
+      "rewards/margins": 0.4368700683116913,
+      "rewards/rejected": -1.1426823139190674,
+      "step": 210
+    },
+    {
+      "epoch": 1.59,
+      "grad_norm": 7.075204680484654,
+      "learning_rate": 2.598314606741573e-07,
+      "logps/chosen": -44.7838249206543,
+      "logps/rejected": -52.02484130859375,
+      "loss": 0.6038,
+      "losses/dpo": 0.5901740193367004,
+      "losses/sft": 1.7182517051696777,
+      "losses/total": 0.5901740193367004,
+      "ref_logps/chosen": -37.16931915283203,
+      "ref_logps/rejected": -41.17018127441406,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.7614503502845764,
+      "rewards/margins": 0.32401591539382935,
+      "rewards/rejected": -1.0854662656784058,
+      "step": 211
+    },
+    {
+      "epoch": 1.6,
+      "grad_norm": 7.444039226905721,
+      "learning_rate": 2.5842696629213486e-07,
+      "logps/chosen": -41.19989776611328,
+      "logps/rejected": -49.64472961425781,
+      "loss": 0.5961,
+      "losses/dpo": 0.5703378319740295,
+      "losses/sft": 1.288915753364563,
+      "losses/total": 0.5703378319740295,
+      "ref_logps/chosen": -34.397457122802734,
+      "ref_logps/rejected": -39.43400573730469,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.6802438497543335,
+      "rewards/margins": 0.34082797169685364,
+      "rewards/rejected": -1.0210717916488647,
+      "step": 212
+    },
+    {
+      "epoch": 1.61,
+      "grad_norm": 7.3272234640712455,
+      "learning_rate": 2.5702247191011236e-07,
+      "logps/chosen": -49.161766052246094,
+      "logps/rejected": -55.671295166015625,
+      "loss": 0.5862,
+      "losses/dpo": 0.7499480843544006,
+      "losses/sft": 1.8793140649795532,
+      "losses/total": 0.7499480843544006,
+      "ref_logps/chosen": -41.297489166259766,
+      "ref_logps/rejected": -44.11161422729492,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7864278554916382,
+      "rewards/margins": 0.3695400655269623,
+      "rewards/rejected": -1.1559679508209229,
+      "step": 213
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 7.817780320058273,
+      "learning_rate": 2.5561797752808987e-07,
+      "logps/chosen": -46.763206481933594,
+      "logps/rejected": -53.703033447265625,
+      "loss": 0.6266,
+      "losses/dpo": 0.48047423362731934,
+      "losses/sft": 1.5680122375488281,
+      "losses/total": 0.48047423362731934,
+      "ref_logps/chosen": -39.49818801879883,
+      "ref_logps/rejected": -43.53962326049805,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.7265015840530396,
+      "rewards/margins": 0.28983935713768005,
+      "rewards/rejected": -1.016340970993042,
+      "step": 214
+    },
+    {
+      "epoch": 1.62,
+      "grad_norm": 7.508782666466954,
+      "learning_rate": 2.5421348314606737e-07,
+      "logps/chosen": -47.78954315185547,
+      "logps/rejected": -56.90927505493164,
+      "loss": 0.5628,
+      "losses/dpo": 0.537736177444458,
+      "losses/sft": 1.6823458671569824,
+      "losses/total": 0.537736177444458,
+      "ref_logps/chosen": -40.28362274169922,
+      "ref_logps/rejected": -45.37610626220703,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.7505923509597778,
+      "rewards/margins": 0.4027244448661804,
+      "rewards/rejected": -1.1533167362213135,
+      "step": 215
+    },
+    {
+      "epoch": 1.63,
+      "grad_norm": 7.806242305852612,
+      "learning_rate": 2.5280898876404493e-07,
+      "logps/chosen": -47.2044677734375,
+      "logps/rejected": -58.619651794433594,
+      "loss": 0.5899,
+      "losses/dpo": 0.6399192214012146,
+      "losses/sft": 1.363295316696167,
+      "losses/total": 0.6399192214012146,
+      "ref_logps/chosen": -39.20256423950195,
+      "ref_logps/rejected": -46.85260772705078,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8001901507377625,
+      "rewards/margins": 0.3765140473842621,
+      "rewards/rejected": -1.1767041683197021,
+      "step": 216
+    },
+    {
+      "epoch": 1.64,
+      "grad_norm": 6.621098009181271,
+      "learning_rate": 2.5140449438202243e-07,
+      "logps/chosen": -36.010169982910156,
+      "logps/rejected": -48.608699798583984,
+      "loss": 0.5501,
+      "losses/dpo": 0.5600734949111938,
+      "losses/sft": 1.3302438259124756,
+      "losses/total": 0.5600734949111938,
+      "ref_logps/chosen": -29.427637100219727,
+      "ref_logps/rejected": -37.72674560546875,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.6582531929016113,
+      "rewards/margins": 0.4299423098564148,
+      "rewards/rejected": -1.088195562362671,
+      "step": 217
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 7.0657521689003735,
+      "learning_rate": 2.5e-07,
+      "logps/chosen": -42.20947265625,
+      "logps/rejected": -53.40728759765625,
+      "loss": 0.5808,
+      "losses/dpo": 0.5706441402435303,
+      "losses/sft": 1.390072226524353,
+      "losses/total": 0.5706441402435303,
+      "ref_logps/chosen": -34.44993591308594,
+      "ref_logps/rejected": -41.737003326416016,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.7759537696838379,
+      "rewards/margins": 0.3910742402076721,
+      "rewards/rejected": -1.1670279502868652,
+      "step": 218
+    },
+    {
+      "epoch": 1.65,
+      "grad_norm": 7.920819614767415,
+      "learning_rate": 2.485955056179775e-07,
+      "logps/chosen": -46.45621109008789,
+      "logps/rejected": -53.37653350830078,
+      "loss": 0.6258,
+      "losses/dpo": 0.5177885293960571,
+      "losses/sft": 1.4505321979522705,
+      "losses/total": 0.5177885293960571,
+      "ref_logps/chosen": -38.40946960449219,
+      "ref_logps/rejected": -42.48009490966797,
+      "rewards/accuracies": 0.6328125,
+      "rewards/chosen": -0.8046744465827942,
+      "rewards/margins": 0.2849688231945038,
+      "rewards/rejected": -1.0896432399749756,
+      "step": 219
+    },
+    {
+      "epoch": 1.66,
+      "grad_norm": 6.775480921328623,
+      "learning_rate": 2.4719101123595505e-07,
+      "logps/chosen": -43.19866943359375,
+      "logps/rejected": -51.142852783203125,
+      "loss": 0.5708,
+      "losses/dpo": 0.631821870803833,
+      "losses/sft": 1.687159776687622,
+      "losses/total": 0.631821870803833,
+      "ref_logps/chosen": -35.93583297729492,
+      "ref_logps/rejected": -40.11430358886719,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.7262836694717407,
+      "rewards/margins": 0.3765709102153778,
+      "rewards/rejected": -1.102854609489441,
+      "step": 220
+    },
+    {
+      "epoch": 1.67,
+      "grad_norm": 7.00534024427554,
+      "learning_rate": 2.4578651685393255e-07,
+      "logps/chosen": -43.32523727416992,
+      "logps/rejected": -52.18841552734375,
+      "loss": 0.56,
+      "losses/dpo": 0.5959673523902893,
+      "losses/sft": 1.5886725187301636,
+      "losses/total": 0.5959673523902893,
+      "ref_logps/chosen": -35.884517669677734,
+      "ref_logps/rejected": -40.54563522338867,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.7440718412399292,
+      "rewards/margins": 0.42020630836486816,
+      "rewards/rejected": -1.1642781496047974,
+      "step": 221
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 7.150852349968827,
+      "learning_rate": 2.443820224719101e-07,
+      "logps/chosen": -42.47400665283203,
+      "logps/rejected": -53.8497314453125,
+      "loss": 0.5456,
+      "losses/dpo": 0.5008928775787354,
+      "losses/sft": 1.4967145919799805,
+      "losses/total": 0.5008928775787354,
+      "ref_logps/chosen": -35.4063720703125,
+      "ref_logps/rejected": -41.911190032958984,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.7067632675170898,
+      "rewards/margins": 0.48709067702293396,
+      "rewards/rejected": -1.1938539743423462,
+      "step": 222
+    },
+    {
+      "epoch": 1.68,
+      "grad_norm": 7.009242529601585,
+      "learning_rate": 2.429775280898876e-07,
+      "logps/chosen": -42.480735778808594,
+      "logps/rejected": -56.96538543701172,
+      "loss": 0.5687,
+      "losses/dpo": 0.5494006872177124,
+      "losses/sft": 1.660073161125183,
+      "losses/total": 0.5494006872177124,
+      "ref_logps/chosen": -34.94923400878906,
+      "ref_logps/rejected": -45.03327178955078,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.753150224685669,
+      "rewards/margins": 0.44006073474884033,
+      "rewards/rejected": -1.1932109594345093,
+      "step": 223
+    },
+    {
+      "epoch": 1.69,
+      "grad_norm": 7.145198782494123,
+      "learning_rate": 2.4157303370786517e-07,
+      "logps/chosen": -46.52253341674805,
+      "logps/rejected": -56.88560485839844,
+      "loss": 0.5578,
+      "losses/dpo": 0.6753450632095337,
+      "losses/sft": 1.733784556388855,
+      "losses/total": 0.6753450632095337,
+      "ref_logps/chosen": -38.87651062011719,
+      "ref_logps/rejected": -44.95452117919922,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.7646023035049438,
+      "rewards/margins": 0.428506076335907,
+      "rewards/rejected": -1.193108320236206,
+      "step": 224
+    },
+    {
+      "epoch": 1.7,
+      "grad_norm": 7.2140738897995895,
+      "learning_rate": 2.401685393258427e-07,
+      "logps/chosen": -44.606842041015625,
+      "logps/rejected": -51.53977966308594,
+      "loss": 0.5755,
+      "losses/dpo": 0.6050464510917664,
+      "losses/sft": 1.4844509363174438,
+      "losses/total": 0.6050464510917664,
+      "ref_logps/chosen": -37.366485595703125,
+      "ref_logps/rejected": -40.17390441894531,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7240351438522339,
+      "rewards/margins": 0.41255253553390503,
+      "rewards/rejected": -1.1365876197814941,
+      "step": 225
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 7.316056461598082,
+      "learning_rate": 2.3876404494382023e-07,
+      "logps/chosen": -43.40976333618164,
+      "logps/rejected": -51.119468688964844,
+      "loss": 0.5856,
+      "losses/dpo": 0.6437182426452637,
+      "losses/sft": 1.6879228353500366,
+      "losses/total": 0.6437182426452637,
+      "ref_logps/chosen": -35.9763298034668,
+      "ref_logps/rejected": -39.941932678222656,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.7433432936668396,
+      "rewards/margins": 0.37441009283065796,
+      "rewards/rejected": -1.1177533864974976,
+      "step": 226
+    },
+    {
+      "epoch": 1.71,
+      "grad_norm": 7.078331857989057,
+      "learning_rate": 2.3735955056179774e-07,
+      "logps/chosen": -45.81120681762695,
+      "logps/rejected": -50.85576629638672,
+      "loss": 0.6076,
+      "losses/dpo": 0.7625922560691833,
+      "losses/sft": 1.5723658800125122,
+      "losses/total": 0.7625922560691833,
+      "ref_logps/chosen": -38.21784973144531,
+      "ref_logps/rejected": -40.007240295410156,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.7593356966972351,
+      "rewards/margins": 0.32551684975624084,
+      "rewards/rejected": -1.0848525762557983,
+      "step": 227
+    },
+    {
+      "epoch": 1.72,
+      "grad_norm": 7.206138039626543,
+      "learning_rate": 2.3595505617977527e-07,
+      "logps/chosen": -43.7403450012207,
+      "logps/rejected": -52.108604431152344,
+      "loss": 0.5922,
+      "losses/dpo": 0.5139514803886414,
+      "losses/sft": 1.6670148372650146,
+      "losses/total": 0.5139514803886414,
+      "ref_logps/chosen": -35.650115966796875,
+      "ref_logps/rejected": -40.582130432128906,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.8090231418609619,
+      "rewards/margins": 0.34362420439720154,
+      "rewards/rejected": -1.1526473760604858,
+      "step": 228
+    },
+    {
+      "epoch": 1.73,
+      "grad_norm": 7.61900579513634,
+      "learning_rate": 2.345505617977528e-07,
+      "logps/chosen": -42.43614959716797,
+      "logps/rejected": -52.779483795166016,
+      "loss": 0.5781,
+      "losses/dpo": 0.4573014974594116,
+      "losses/sft": 1.5003488063812256,
+      "losses/total": 0.4573014974594116,
+      "ref_logps/chosen": -34.90065002441406,
+      "ref_logps/rejected": -41.4276237487793,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.7535501718521118,
+      "rewards/margins": 0.38163578510284424,
+      "rewards/rejected": -1.135185956954956,
+      "step": 229
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 7.829509763007773,
+      "learning_rate": 2.331460674157303e-07,
+      "logps/chosen": -47.7276496887207,
+      "logps/rejected": -56.36402893066406,
+      "loss": 0.5302,
+      "losses/dpo": 0.529563307762146,
+      "losses/sft": 1.6256301403045654,
+      "losses/total": 0.529563307762146,
+      "ref_logps/chosen": -40.326351165771484,
+      "ref_logps/rejected": -43.704612731933594,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.740129828453064,
+      "rewards/margins": 0.5258119702339172,
+      "rewards/rejected": -1.2659417390823364,
+      "step": 230
+    },
+    {
+      "epoch": 1.74,
+      "grad_norm": 8.06687120109026,
+      "learning_rate": 2.3174157303370786e-07,
+      "logps/chosen": -44.74425506591797,
+      "logps/rejected": -55.536312103271484,
+      "loss": 0.561,
+      "losses/dpo": 0.4758527874946594,
+      "losses/sft": 1.3779159784317017,
+      "losses/total": 0.4758527874946594,
+      "ref_logps/chosen": -37.057281494140625,
+      "ref_logps/rejected": -43.34575653076172,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.7686972618103027,
+      "rewards/margins": 0.45035821199417114,
+      "rewards/rejected": -1.2190555334091187,
+      "step": 231
+    },
+    {
+      "epoch": 1.75,
+      "grad_norm": 7.291686332994008,
+      "learning_rate": 2.303370786516854e-07,
+      "logps/chosen": -43.00548553466797,
+      "logps/rejected": -54.49897003173828,
+      "loss": 0.5834,
+      "losses/dpo": 0.5421339273452759,
+      "losses/sft": 1.4051011800765991,
+      "losses/total": 0.5421339273452759,
+      "ref_logps/chosen": -35.81233215332031,
+      "ref_logps/rejected": -43.77638244628906,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7193150520324707,
+      "rewards/margins": 0.35294392704963684,
+      "rewards/rejected": -1.0722589492797852,
+      "step": 232
+    },
+    {
+      "epoch": 1.76,
+      "grad_norm": 6.669119014424567,
+      "learning_rate": 2.2893258426966292e-07,
+      "logps/chosen": -42.595909118652344,
+      "logps/rejected": -50.517574310302734,
+      "loss": 0.5942,
+      "losses/dpo": 0.5890272855758667,
+      "losses/sft": 1.3421604633331299,
+      "losses/total": 0.5890272855758667,
+      "ref_logps/chosen": -35.23419189453125,
+      "ref_logps/rejected": -39.80196762084961,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.7361720204353333,
+      "rewards/margins": 0.33538877964019775,
+      "rewards/rejected": -1.0715608596801758,
+      "step": 233
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 7.634998140383259,
+      "learning_rate": 2.2752808988764045e-07,
+      "logps/chosen": -48.97822189331055,
+      "logps/rejected": -55.01988983154297,
+      "loss": 0.6041,
+      "losses/dpo": 0.4961914122104645,
+      "losses/sft": 1.6347143650054932,
+      "losses/total": 0.4961914122104645,
+      "ref_logps/chosen": -40.457027435302734,
+      "ref_logps/rejected": -43.25347900390625,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.8521193265914917,
+      "rewards/margins": 0.3245222866535187,
+      "rewards/rejected": -1.1766417026519775,
+      "step": 234
+    },
+    {
+      "epoch": 1.77,
+      "grad_norm": 7.773471295460603,
+      "learning_rate": 2.2612359550561795e-07,
+      "logps/chosen": -46.672576904296875,
+      "logps/rejected": -54.91902542114258,
+      "loss": 0.5883,
+      "losses/dpo": 0.5112382173538208,
+      "losses/sft": 1.6069546937942505,
+      "losses/total": 0.5112382173538208,
+      "ref_logps/chosen": -38.114097595214844,
+      "ref_logps/rejected": -42.68096160888672,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.8558481931686401,
+      "rewards/margins": 0.36795809864997864,
+      "rewards/rejected": -1.2238062620162964,
+      "step": 235
+    },
+    {
+      "epoch": 1.78,
+      "grad_norm": 6.9224359595021925,
+      "learning_rate": 2.2471910112359549e-07,
+      "logps/chosen": -43.26789855957031,
+      "logps/rejected": -49.846065521240234,
+      "loss": 0.5976,
+      "losses/dpo": 0.530718207359314,
+      "losses/sft": 1.4825395345687866,
+      "losses/total": 0.530718207359314,
+      "ref_logps/chosen": -35.522216796875,
+      "ref_logps/rejected": -38.95735168457031,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.7745683193206787,
+      "rewards/margins": 0.3143025040626526,
+      "rewards/rejected": -1.088870882987976,
+      "step": 236
+    },
+    {
+      "epoch": 1.79,
+      "grad_norm": 7.5835387702946075,
+      "learning_rate": 2.2331460674157302e-07,
+      "logps/chosen": -44.358116149902344,
+      "logps/rejected": -57.51253890991211,
+      "loss": 0.5667,
+      "losses/dpo": 0.4763038754463196,
+      "losses/sft": 1.4994385242462158,
+      "losses/total": 0.4763038754463196,
+      "ref_logps/chosen": -36.64021682739258,
+      "ref_logps/rejected": -45.455902099609375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7717897295951843,
+      "rewards/margins": 0.4338740408420563,
+      "rewards/rejected": -1.205663800239563,
+      "step": 237
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 6.860345248290717,
+      "learning_rate": 2.2191011235955055e-07,
+      "logps/chosen": -43.448211669921875,
+      "logps/rejected": -52.67967224121094,
+      "loss": 0.5841,
+      "losses/dpo": 0.5856455564498901,
+      "losses/sft": 1.5493735074996948,
+      "losses/total": 0.5856455564498901,
+      "ref_logps/chosen": -35.36868667602539,
+      "ref_logps/rejected": -40.977325439453125,
+      "rewards/accuracies": 0.65625,
+      "rewards/chosen": -0.8079524040222168,
+      "rewards/margins": 0.36228203773498535,
+      "rewards/rejected": -1.1702344417572021,
+      "step": 238
+    },
+    {
+      "epoch": 1.8,
+      "grad_norm": 6.877362645097382,
+      "learning_rate": 2.205056179775281e-07,
+      "logps/chosen": -43.96727752685547,
+      "logps/rejected": -54.08544921875,
+      "loss": 0.561,
+      "losses/dpo": 0.7524189352989197,
+      "losses/sft": 1.4943475723266602,
+      "losses/total": 0.7524189352989197,
+      "ref_logps/chosen": -36.482398986816406,
+      "ref_logps/rejected": -42.11739730834961,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.7484874725341797,
+      "rewards/margins": 0.44831788539886475,
+      "rewards/rejected": -1.1968053579330444,
+      "step": 239
+    },
+    {
+      "epoch": 1.81,
+      "grad_norm": 8.176287850888796,
+      "learning_rate": 2.191011235955056e-07,
+      "logps/chosen": -44.187679290771484,
+      "logps/rejected": -52.56245422363281,
+      "loss": 0.6215,
+      "losses/dpo": 0.5882298946380615,
+      "losses/sft": 1.509756326675415,
+      "losses/total": 0.5882298946380615,
+      "ref_logps/chosen": -35.26359558105469,
+      "ref_logps/rejected": -40.418216705322266,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.8924084901809692,
+      "rewards/margins": 0.32201528549194336,
+      "rewards/rejected": -1.2144238948822021,
+      "step": 240
+    },
+    {
+      "epoch": 1.82,
+      "grad_norm": 6.748177391589757,
+      "learning_rate": 2.1769662921348314e-07,
+      "logps/chosen": -44.592193603515625,
+      "logps/rejected": -54.5892219543457,
+      "loss": 0.5454,
+      "losses/dpo": 0.5010501742362976,
+      "losses/sft": 1.692970871925354,
+      "losses/total": 0.5010501742362976,
+      "ref_logps/chosen": -36.254737854003906,
+      "ref_logps/rejected": -41.49382019042969,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.8337457180023193,
+      "rewards/margins": 0.47579440474510193,
+      "rewards/rejected": -1.3095402717590332,
+      "step": 241
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 7.9524609339083385,
+      "learning_rate": 2.1629213483146067e-07,
+      "logps/chosen": -49.09219741821289,
+      "logps/rejected": -55.76482391357422,
+      "loss": 0.6033,
+      "losses/dpo": 0.5440762042999268,
+      "losses/sft": 1.7360166311264038,
+      "losses/total": 0.5440762042999268,
+      "ref_logps/chosen": -39.933494567871094,
+      "ref_logps/rejected": -43.29194259643555,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.915870189666748,
+      "rewards/margins": 0.33141782879829407,
+      "rewards/rejected": -1.2472879886627197,
+      "step": 242
+    },
+    {
+      "epoch": 1.83,
+      "grad_norm": 7.715847211838437,
+      "learning_rate": 2.148876404494382e-07,
+      "logps/chosen": -44.42055130004883,
+      "logps/rejected": -50.31025314331055,
+      "loss": 0.6419,
+      "losses/dpo": 0.6423018574714661,
+      "losses/sft": 1.8698339462280273,
+      "losses/total": 0.6423018574714661,
+      "ref_logps/chosen": -36.14445114135742,
+      "ref_logps/rejected": -39.49055480957031,
+      "rewards/accuracies": 0.640625,
+      "rewards/chosen": -0.8276099562644958,
+      "rewards/margins": 0.2543600797653198,
+      "rewards/rejected": -1.081969976425171,
+      "step": 243
+    },
+    {
+      "epoch": 1.84,
+      "grad_norm": 7.717852694753547,
+      "learning_rate": 2.134831460674157e-07,
+      "logps/chosen": -45.71333694458008,
+      "logps/rejected": -56.663360595703125,
+      "loss": 0.5667,
+      "losses/dpo": 0.578036904335022,
+      "losses/sft": 1.548266053199768,
+      "losses/total": 0.578036904335022,
+      "ref_logps/chosen": -37.3514404296875,
+      "ref_logps/rejected": -44.125831604003906,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.8361901044845581,
+      "rewards/margins": 0.4175630807876587,
+      "rewards/rejected": -1.2537531852722168,
+      "step": 244
+    },
+    {
+      "epoch": 1.85,
+      "grad_norm": 7.217778469739938,
+      "learning_rate": 2.1207865168539323e-07,
+      "logps/chosen": -47.56121826171875,
+      "logps/rejected": -55.15635299682617,
+      "loss": 0.584,
+      "losses/dpo": 0.6499341726303101,
+      "losses/sft": 1.8146308660507202,
+      "losses/total": 0.6499341726303101,
+      "ref_logps/chosen": -39.167484283447266,
+      "ref_logps/rejected": -43.06206130981445,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8393731117248535,
+      "rewards/margins": 0.37005579471588135,
+      "rewards/rejected": -1.2094289064407349,
+      "step": 245
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 7.706531034729977,
+      "learning_rate": 2.1067415730337076e-07,
+      "logps/chosen": -45.61647415161133,
+      "logps/rejected": -55.11760330200195,
+      "loss": 0.616,
+      "losses/dpo": 0.45806318521499634,
+      "losses/sft": 1.4561158418655396,
+      "losses/total": 0.45806318521499634,
+      "ref_logps/chosen": -36.999488830566406,
+      "ref_logps/rejected": -43.26171112060547,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8616988062858582,
+      "rewards/margins": 0.3238902986049652,
+      "rewards/rejected": -1.185589075088501,
+      "step": 246
+    },
+    {
+      "epoch": 1.86,
+      "grad_norm": 7.805010195142929,
+      "learning_rate": 2.0926966292134832e-07,
+      "logps/chosen": -43.88758087158203,
+      "logps/rejected": -54.20425033569336,
+      "loss": 0.5977,
+      "losses/dpo": 0.6152101755142212,
+      "losses/sft": 1.5027949810028076,
+      "losses/total": 0.6152101755142212,
+      "ref_logps/chosen": -35.95214080810547,
+      "ref_logps/rejected": -42.84498977661133,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7935442924499512,
+      "rewards/margins": 0.3423812687397003,
+      "rewards/rejected": -1.135925531387329,
+      "step": 247
+    },
+    {
+      "epoch": 1.87,
+      "grad_norm": 7.406497027707841,
+      "learning_rate": 2.0786516853932585e-07,
+      "logps/chosen": -45.788818359375,
+      "logps/rejected": -53.047203063964844,
+      "loss": 0.5831,
+      "losses/dpo": 0.4654901325702667,
+      "losses/sft": 1.479446291923523,
+      "losses/total": 0.4654901325702667,
+      "ref_logps/chosen": -37.57151794433594,
+      "ref_logps/rejected": -41.39836883544922,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.8217304348945618,
+      "rewards/margins": 0.34315240383148193,
+      "rewards/rejected": -1.1648828983306885,
+      "step": 248
+    },
+    {
+      "epoch": 1.88,
+      "grad_norm": 7.682816189246604,
+      "learning_rate": 2.0646067415730336e-07,
+      "logps/chosen": -45.08941650390625,
+      "logps/rejected": -56.03681182861328,
+      "loss": 0.6254,
+      "losses/dpo": 0.6420303583145142,
+      "losses/sft": 1.767283320426941,
+      "losses/total": 0.6420303583145142,
+      "ref_logps/chosen": -35.47539138793945,
+      "ref_logps/rejected": -43.610809326171875,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.9614025950431824,
+      "rewards/margins": 0.281198114156723,
+      "rewards/rejected": -1.242600679397583,
+      "step": 249
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 7.381909400967013,
+      "learning_rate": 2.0505617977528089e-07,
+      "logps/chosen": -44.02425765991211,
+      "logps/rejected": -57.3465461730957,
+      "loss": 0.552,
+      "losses/dpo": 0.5316831469535828,
+      "losses/sft": 1.4193787574768066,
+      "losses/total": 0.5316831469535828,
+      "ref_logps/chosen": -36.48744201660156,
+      "ref_logps/rejected": -45.4715576171875,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7536818385124207,
+      "rewards/margins": 0.43381738662719727,
+      "rewards/rejected": -1.1874991655349731,
+      "step": 250
+    },
+    {
+      "epoch": 1.89,
+      "grad_norm": 7.457988261963576,
+      "learning_rate": 2.0365168539325842e-07,
+      "logps/chosen": -44.05774688720703,
+      "logps/rejected": -54.26824951171875,
+      "loss": 0.5622,
+      "losses/dpo": 0.6149340867996216,
+      "losses/sft": 1.7144936323165894,
+      "losses/total": 0.6149340867996216,
+      "ref_logps/chosen": -35.81959533691406,
+      "ref_logps/rejected": -41.808128356933594,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.8238149881362915,
+      "rewards/margins": 0.4221975803375244,
+      "rewards/rejected": -1.246012568473816,
+      "step": 251
+    },
+    {
+      "epoch": 1.9,
+      "grad_norm": 6.886174599694686,
+      "learning_rate": 2.0224719101123595e-07,
+      "logps/chosen": -42.96266174316406,
+      "logps/rejected": -57.41224670410156,
+      "loss": 0.5338,
+      "losses/dpo": 0.5937738418579102,
+      "losses/sft": 1.7894150018692017,
+      "losses/total": 0.5937738418579102,
+      "ref_logps/chosen": -35.350067138671875,
+      "ref_logps/rejected": -44.520416259765625,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.7612596750259399,
+      "rewards/margins": 0.5279234051704407,
+      "rewards/rejected": -1.2891831398010254,
+      "step": 252
+    },
+    {
+      "epoch": 1.91,
+      "grad_norm": 7.229890008822798,
+      "learning_rate": 2.0084269662921348e-07,
+      "logps/chosen": -40.15806198120117,
+      "logps/rejected": -51.43259811401367,
+      "loss": 0.5704,
+      "losses/dpo": 0.6577882170677185,
+      "losses/sft": 1.8345617055892944,
+      "losses/total": 0.6577882170677185,
+      "ref_logps/chosen": -32.859107971191406,
+      "ref_logps/rejected": -40.261077880859375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7298952341079712,
+      "rewards/margins": 0.3872564733028412,
+      "rewards/rejected": -1.1171517372131348,
+      "step": 253
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 7.4822334369379995,
+      "learning_rate": 1.9943820224719098e-07,
+      "logps/chosen": -47.686946868896484,
+      "logps/rejected": -57.150779724121094,
+      "loss": 0.5379,
+      "losses/dpo": 0.5903155207633972,
+      "losses/sft": 1.7529627084732056,
+      "losses/total": 0.5903155207633972,
+      "ref_logps/chosen": -39.64442443847656,
+      "ref_logps/rejected": -44.537864685058594,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8042521476745605,
+      "rewards/margins": 0.45703911781311035,
+      "rewards/rejected": -1.2612911462783813,
+      "step": 254
+    },
+    {
+      "epoch": 1.92,
+      "grad_norm": 8.026318217758316,
+      "learning_rate": 1.9803370786516854e-07,
+      "logps/chosen": -48.09050750732422,
+      "logps/rejected": -55.54762268066406,
+      "loss": 0.6168,
+      "losses/dpo": 0.6026707887649536,
+      "losses/sft": 1.538877248764038,
+      "losses/total": 0.6026707887649536,
+      "ref_logps/chosen": -39.93636703491211,
+      "ref_logps/rejected": -44.63798522949219,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.8154144287109375,
+      "rewards/margins": 0.2755492627620697,
+      "rewards/rejected": -1.0909637212753296,
+      "step": 255
+    },
+    {
+      "epoch": 1.93,
+      "grad_norm": 7.789382763460605,
+      "learning_rate": 1.9662921348314607e-07,
+      "logps/chosen": -42.904396057128906,
+      "logps/rejected": -52.95304489135742,
+      "loss": 0.6004,
+      "losses/dpo": 0.6533941626548767,
+      "losses/sft": 1.7555681467056274,
+      "losses/total": 0.6533941626548767,
+      "ref_logps/chosen": -35.12152862548828,
+      "ref_logps/rejected": -41.70171356201172,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.7782862186431885,
+      "rewards/margins": 0.34684672951698303,
+      "rewards/rejected": -1.1251329183578491,
+      "step": 256
+    },
+    {
+      "epoch": 1.94,
+      "grad_norm": 7.650755358628509,
+      "learning_rate": 1.952247191011236e-07,
+      "logps/chosen": -47.547119140625,
+      "logps/rejected": -55.00044250488281,
+      "loss": 0.5857,
+      "losses/dpo": 0.533769965171814,
+      "losses/sft": 1.518601655960083,
+      "losses/total": 0.533769965171814,
+      "ref_logps/chosen": -39.58103942871094,
+      "ref_logps/rejected": -43.044471740722656,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7966080904006958,
+      "rewards/margins": 0.3989890217781067,
+      "rewards/rejected": -1.1955971717834473,
+      "step": 257
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 7.218761250045399,
+      "learning_rate": 1.938202247191011e-07,
+      "logps/chosen": -45.877933502197266,
+      "logps/rejected": -55.09804916381836,
+      "loss": 0.5628,
+      "losses/dpo": 0.5916406512260437,
+      "losses/sft": 1.787639856338501,
+      "losses/total": 0.5916406512260437,
+      "ref_logps/chosen": -37.8803596496582,
+      "ref_logps/rejected": -42.772945404052734,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.7997570633888245,
+      "rewards/margins": 0.43275338411331177,
+      "rewards/rejected": -1.2325104475021362,
+      "step": 258
+    },
+    {
+      "epoch": 1.95,
+      "grad_norm": 6.746342603050737,
+      "learning_rate": 1.9241573033707863e-07,
+      "logps/chosen": -44.5426139831543,
+      "logps/rejected": -52.97711944580078,
+      "loss": 0.5404,
+      "losses/dpo": 0.6297707557678223,
+      "losses/sft": 1.9339282512664795,
+      "losses/total": 0.6297707557678223,
+      "ref_logps/chosen": -36.8856315612793,
+      "ref_logps/rejected": -40.58320617675781,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.7656983137130737,
+      "rewards/margins": 0.47369277477264404,
+      "rewards/rejected": -1.2393909692764282,
+      "step": 259
+    },
+    {
+      "epoch": 1.96,
+      "grad_norm": 7.487018325482117,
+      "learning_rate": 1.9101123595505617e-07,
+      "logps/chosen": -42.5137825012207,
+      "logps/rejected": -53.166908264160156,
+      "loss": 0.5707,
+      "losses/dpo": 0.7028491497039795,
+      "losses/sft": 1.704848289489746,
+      "losses/total": 0.7028491497039795,
+      "ref_logps/chosen": -34.8709716796875,
+      "ref_logps/rejected": -41.41071701049805,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7642812728881836,
+      "rewards/margins": 0.4113379120826721,
+      "rewards/rejected": -1.1756192445755005,
+      "step": 260
+    },
+    {
+      "epoch": 1.97,
+      "grad_norm": 6.897909731781275,
+      "learning_rate": 1.896067415730337e-07,
+      "logps/chosen": -42.64958190917969,
+      "logps/rejected": -54.01194763183594,
+      "loss": 0.5508,
+      "losses/dpo": 0.6007636785507202,
+      "losses/sft": 1.6722173690795898,
+      "losses/total": 0.6007636785507202,
+      "ref_logps/chosen": -34.81106948852539,
+      "ref_logps/rejected": -41.885292053222656,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7838513851165771,
+      "rewards/margins": 0.428814560174942,
+      "rewards/rejected": -1.2126659154891968,
+      "step": 261
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 6.802922485274152,
+      "learning_rate": 1.8820224719101123e-07,
+      "logps/chosen": -40.00798034667969,
+      "logps/rejected": -54.30394744873047,
+      "loss": 0.5499,
+      "losses/dpo": 0.39324456453323364,
+      "losses/sft": 1.4311751127243042,
+      "losses/total": 0.39324456453323364,
+      "ref_logps/chosen": -32.748207092285156,
+      "ref_logps/rejected": -42.49259948730469,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.7259770035743713,
+      "rewards/margins": 0.4551584720611572,
+      "rewards/rejected": -1.1811354160308838,
+      "step": 262
+    },
+    {
+      "epoch": 1.98,
+      "grad_norm": 6.783836899709174,
+      "learning_rate": 1.8679775280898876e-07,
+      "logps/chosen": -39.83095932006836,
+      "logps/rejected": -54.880165100097656,
+      "loss": 0.5218,
+      "losses/dpo": 0.5562885999679565,
+      "losses/sft": 1.581786036491394,
+      "losses/total": 0.5562885999679565,
+      "ref_logps/chosen": -32.503440856933594,
+      "ref_logps/rejected": -42.00752258300781,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.7327523231506348,
+      "rewards/margins": 0.5545117259025574,
+      "rewards/rejected": -1.287264108657837,
+      "step": 263
+    },
+    {
+      "epoch": 1.99,
+      "grad_norm": 7.863916442503097,
+      "learning_rate": 1.853932584269663e-07,
+      "logps/chosen": -50.77809524536133,
+      "logps/rejected": -57.57705307006836,
+      "loss": 0.5746,
+      "losses/dpo": 0.5502392053604126,
+      "losses/sft": 1.671476125717163,
+      "losses/total": 0.5502392053604126,
+      "ref_logps/chosen": -42.0257568359375,
+      "ref_logps/rejected": -44.61543273925781,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8752338886260986,
+      "rewards/margins": 0.42092812061309814,
+      "rewards/rejected": -1.2961618900299072,
+      "step": 264
+    },
+    {
+      "epoch": 2.0,
+      "grad_norm": 7.397057411594154,
+      "learning_rate": 1.8398876404494382e-07,
+      "logps/chosen": -45.31150817871094,
+      "logps/rejected": -54.171669006347656,
+      "loss": 0.5905,
+      "losses/dpo": 0.5974111557006836,
+      "losses/sft": 1.7264142036437988,
+      "losses/total": 0.5974111557006836,
+      "ref_logps/chosen": -36.78529357910156,
+      "ref_logps/rejected": -41.80047607421875,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.8526214361190796,
+      "rewards/margins": 0.3844982385635376,
+      "rewards/rejected": -1.2371195554733276,
+      "step": 265
+    },
+    {
+      "epoch": 2.01,
+      "grad_norm": 7.017591256124865,
+      "learning_rate": 1.8258426966292135e-07,
+      "logps/chosen": -43.99406433105469,
+      "logps/rejected": -53.245262145996094,
+      "loss": 0.5349,
+      "losses/dpo": 0.5464926362037659,
+      "losses/sft": 1.5807067155838013,
+      "losses/total": 0.5464926362037659,
+      "ref_logps/chosen": -36.38019561767578,
+      "ref_logps/rejected": -40.543190002441406,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.761387288570404,
+      "rewards/margins": 0.5088198184967041,
+      "rewards/rejected": -1.2702070474624634,
+      "step": 266
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 6.985219378495969,
+      "learning_rate": 1.8117977528089888e-07,
+      "logps/chosen": -44.4912109375,
+      "logps/rejected": -53.347251892089844,
+      "loss": 0.5743,
+      "losses/dpo": 0.5905557870864868,
+      "losses/sft": 1.7700715065002441,
+      "losses/total": 0.5905557870864868,
+      "ref_logps/chosen": -36.70296859741211,
+      "ref_logps/rejected": -41.48924255371094,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7788243889808655,
+      "rewards/margins": 0.40697669982910156,
+      "rewards/rejected": -1.1858012676239014,
+      "step": 267
+    },
+    {
+      "epoch": 2.02,
+      "grad_norm": 7.021871568299538,
+      "learning_rate": 1.7977528089887638e-07,
+      "logps/chosen": -41.247291564941406,
+      "logps/rejected": -50.476539611816406,
+      "loss": 0.5583,
+      "losses/dpo": 0.5536283850669861,
+      "losses/sft": 1.3929085731506348,
+      "losses/total": 0.5536283850669861,
+      "ref_logps/chosen": -34.45301055908203,
+      "ref_logps/rejected": -39.05378723144531,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.6794286370277405,
+      "rewards/margins": 0.46284645795822144,
+      "rewards/rejected": -1.142275094985962,
+      "step": 268
+    },
+    {
+      "epoch": 2.03,
+      "grad_norm": 7.290879700745406,
+      "learning_rate": 1.7837078651685391e-07,
+      "logps/chosen": -44.69060516357422,
+      "logps/rejected": -52.723419189453125,
+      "loss": 0.5862,
+      "losses/dpo": 0.5445826053619385,
+      "losses/sft": 1.8489296436309814,
+      "losses/total": 0.5445826053619385,
+      "ref_logps/chosen": -36.05701446533203,
+      "ref_logps/rejected": -40.36290740966797,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.863358736038208,
+      "rewards/margins": 0.37269291281700134,
+      "rewards/rejected": -1.2360515594482422,
+      "step": 269
+    },
+    {
+      "epoch": 2.04,
+      "grad_norm": 7.278112500918291,
+      "learning_rate": 1.7696629213483144e-07,
+      "logps/chosen": -47.17387771606445,
+      "logps/rejected": -55.31304168701172,
+      "loss": 0.5451,
+      "losses/dpo": 0.4929129481315613,
+      "losses/sft": 1.2738251686096191,
+      "losses/total": 0.4929129481315613,
+      "ref_logps/chosen": -38.97121810913086,
+      "ref_logps/rejected": -42.48224639892578,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.8202658891677856,
+      "rewards/margins": 0.4628136157989502,
+      "rewards/rejected": -1.2830795049667358,
+      "step": 270
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 7.2790694246,
+      "learning_rate": 1.75561797752809e-07,
+      "logps/chosen": -39.39592742919922,
+      "logps/rejected": -49.23228073120117,
+      "loss": 0.5839,
+      "losses/dpo": 0.5500213503837585,
+      "losses/sft": 1.5326621532440186,
+      "losses/total": 0.5500213503837585,
+      "ref_logps/chosen": -31.87863540649414,
+      "ref_logps/rejected": -37.98380661010742,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.7517290115356445,
+      "rewards/margins": 0.37311792373657227,
+      "rewards/rejected": -1.1248469352722168,
+      "step": 271
+    },
+    {
+      "epoch": 2.05,
+      "grad_norm": 6.952262857114201,
+      "learning_rate": 1.741573033707865e-07,
+      "logps/chosen": -41.979820251464844,
+      "logps/rejected": -51.27606964111328,
+      "loss": 0.5477,
+      "losses/dpo": 0.5781035423278809,
+      "losses/sft": 1.6893967390060425,
+      "losses/total": 0.5781035423278809,
+      "ref_logps/chosen": -34.47309875488281,
+      "ref_logps/rejected": -38.76087951660156,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.7506722211837769,
+      "rewards/margins": 0.5008465051651001,
+      "rewards/rejected": -1.251518726348877,
+      "step": 272
+    },
+    {
+      "epoch": 2.06,
+      "grad_norm": 7.323213695486467,
+      "learning_rate": 1.7275280898876404e-07,
+      "logps/chosen": -46.38153839111328,
+      "logps/rejected": -57.915809631347656,
+      "loss": 0.5002,
+      "losses/dpo": 0.5204892754554749,
+      "losses/sft": 1.5103009939193726,
+      "losses/total": 0.5204892754554749,
+      "ref_logps/chosen": -38.351890563964844,
+      "ref_logps/rejected": -43.90599060058594,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.8029646873474121,
+      "rewards/margins": 0.5980167388916016,
+      "rewards/rejected": -1.4009814262390137,
+      "step": 273
+    },
+    {
+      "epoch": 2.07,
+      "grad_norm": 7.764155916683402,
+      "learning_rate": 1.7134831460674157e-07,
+      "logps/chosen": -45.19919967651367,
+      "logps/rejected": -51.13863754272461,
+      "loss": 0.6283,
+      "losses/dpo": 0.616185188293457,
+      "losses/sft": 1.6811277866363525,
+      "losses/total": 0.616185188293457,
+      "ref_logps/chosen": -36.72953796386719,
+      "ref_logps/rejected": -39.85737228393555,
+      "rewards/accuracies": 0.6171875,
+      "rewards/chosen": -0.8469663858413696,
+      "rewards/margins": 0.2811599373817444,
+      "rewards/rejected": -1.1281262636184692,
+      "step": 274
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 7.020433782892144,
+      "learning_rate": 1.699438202247191e-07,
+      "logps/chosen": -43.26371765136719,
+      "logps/rejected": -52.273712158203125,
+      "loss": 0.5707,
+      "losses/dpo": 0.584823727607727,
+      "losses/sft": 1.7780404090881348,
+      "losses/total": 0.584823727607727,
+      "ref_logps/chosen": -34.843936920166016,
+      "ref_logps/rejected": -39.69860076904297,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.8419777154922485,
+      "rewards/margins": 0.4155334234237671,
+      "rewards/rejected": -1.2575111389160156,
+      "step": 275
+    },
+    {
+      "epoch": 2.08,
+      "grad_norm": 7.92264626489854,
+      "learning_rate": 1.6853932584269663e-07,
+      "logps/chosen": -47.98881912231445,
+      "logps/rejected": -56.07038116455078,
+      "loss": 0.5876,
+      "losses/dpo": 0.4782869219779968,
+      "losses/sft": 1.5796866416931152,
+      "losses/total": 0.4782869219779968,
+      "ref_logps/chosen": -39.32164764404297,
+      "ref_logps/rejected": -43.3940315246582,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.8667174577713013,
+      "rewards/margins": 0.40091750025749207,
+      "rewards/rejected": -1.2676348686218262,
+      "step": 276
+    },
+    {
+      "epoch": 2.09,
+      "grad_norm": 6.857885259771192,
+      "learning_rate": 1.6713483146067413e-07,
+      "logps/chosen": -42.90391159057617,
+      "logps/rejected": -53.66696548461914,
+      "loss": 0.5666,
+      "losses/dpo": 0.6092857122421265,
+      "losses/sft": 1.6311126947402954,
+      "losses/total": 0.6092857122421265,
+      "ref_logps/chosen": -35.07619857788086,
+      "ref_logps/rejected": -41.65775680541992,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.7827714085578918,
+      "rewards/margins": 0.4181497395038605,
+      "rewards/rejected": -1.2009210586547852,
+      "step": 277
+    },
+    {
+      "epoch": 2.1,
+      "grad_norm": 7.0081480343548215,
+      "learning_rate": 1.6573033707865166e-07,
+      "logps/chosen": -43.48851013183594,
+      "logps/rejected": -56.560142517089844,
+      "loss": 0.5552,
+      "losses/dpo": 0.47724148631095886,
+      "losses/sft": 1.4892723560333252,
+      "losses/total": 0.47724148631095886,
+      "ref_logps/chosen": -35.48023223876953,
+      "ref_logps/rejected": -43.66110610961914,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.8008283376693726,
+      "rewards/margins": 0.4890754222869873,
+      "rewards/rejected": -1.2899038791656494,
+      "step": 278
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 7.160682409155752,
+      "learning_rate": 1.6432584269662922e-07,
+      "logps/chosen": -44.21363830566406,
+      "logps/rejected": -58.09941864013672,
+      "loss": 0.5245,
+      "losses/dpo": 0.4419279396533966,
+      "losses/sft": 1.6377503871917725,
+      "losses/total": 0.4419279396533966,
+      "ref_logps/chosen": -36.17689514160156,
+      "ref_logps/rejected": -44.57625198364258,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8036742210388184,
+      "rewards/margins": 0.5486425757408142,
+      "rewards/rejected": -1.3523168563842773,
+      "step": 279
+    },
+    {
+      "epoch": 2.11,
+      "grad_norm": 7.212324015356224,
+      "learning_rate": 1.6292134831460675e-07,
+      "logps/chosen": -44.50836181640625,
+      "logps/rejected": -51.84413528442383,
+      "loss": 0.5731,
+      "losses/dpo": 0.4934471547603607,
+      "losses/sft": 1.4699177742004395,
+      "losses/total": 0.4934471547603607,
+      "ref_logps/chosen": -36.38746643066406,
+      "ref_logps/rejected": -39.50672149658203,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.8120898604393005,
+      "rewards/margins": 0.4216514825820923,
+      "rewards/rejected": -1.233741283416748,
+      "step": 280
+    },
+    {
+      "epoch": 2.12,
+      "grad_norm": 6.532989304583127,
+      "learning_rate": 1.6151685393258428e-07,
+      "logps/chosen": -40.479827880859375,
+      "logps/rejected": -52.349693298339844,
+      "loss": 0.5111,
+      "losses/dpo": 0.5451053380966187,
+      "losses/sft": 1.5731171369552612,
+      "losses/total": 0.5451053380966187,
+      "ref_logps/chosen": -33.727699279785156,
+      "ref_logps/rejected": -40.0911865234375,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.6752126216888428,
+      "rewards/margins": 0.5506378412246704,
+      "rewards/rejected": -1.2258504629135132,
+      "step": 281
+    },
+    {
+      "epoch": 2.13,
+      "grad_norm": 6.65644378559116,
+      "learning_rate": 1.6011235955056178e-07,
+      "logps/chosen": -42.37626266479492,
+      "logps/rejected": -53.92717742919922,
+      "loss": 0.5305,
+      "losses/dpo": 0.523646354675293,
+      "losses/sft": 1.5671043395996094,
+      "losses/total": 0.523646354675293,
+      "ref_logps/chosen": -35.16848373413086,
+      "ref_logps/rejected": -41.62503433227539,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.7207781076431274,
+      "rewards/margins": 0.509436309337616,
+      "rewards/rejected": -1.2302143573760986,
+      "step": 282
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 7.652354022803428,
+      "learning_rate": 1.5870786516853931e-07,
+      "logps/chosen": -45.83367919921875,
+      "logps/rejected": -57.50337219238281,
+      "loss": 0.5511,
+      "losses/dpo": 0.5657609105110168,
+      "losses/sft": 1.511309266090393,
+      "losses/total": 0.5657609105110168,
+      "ref_logps/chosen": -37.572113037109375,
+      "ref_logps/rejected": -44.38706970214844,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.8261568546295166,
+      "rewards/margins": 0.4854734539985657,
+      "rewards/rejected": -1.3116302490234375,
+      "step": 283
+    },
+    {
+      "epoch": 2.14,
+      "grad_norm": 6.7860253446718,
+      "learning_rate": 1.5730337078651685e-07,
+      "logps/chosen": -41.25431823730469,
+      "logps/rejected": -54.90302658081055,
+      "loss": 0.5188,
+      "losses/dpo": 0.5078562498092651,
+      "losses/sft": 1.5500166416168213,
+      "losses/total": 0.5078562498092651,
+      "ref_logps/chosen": -33.7675666809082,
+      "ref_logps/rejected": -41.983367919921875,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.7486748695373535,
+      "rewards/margins": 0.5432910919189453,
+      "rewards/rejected": -1.2919659614562988,
+      "step": 284
+    },
+    {
+      "epoch": 2.15,
+      "grad_norm": 7.11903493041396,
+      "learning_rate": 1.5589887640449438e-07,
+      "logps/chosen": -43.894989013671875,
+      "logps/rejected": -58.60367202758789,
+      "loss": 0.4914,
+      "losses/dpo": 0.5028943419456482,
+      "losses/sft": 1.594357967376709,
+      "losses/total": 0.5028943419456482,
+      "ref_logps/chosen": -36.30883026123047,
+      "ref_logps/rejected": -45.05226516723633,
+      "rewards/accuracies": 0.8359375,
+      "rewards/chosen": -0.7586159706115723,
+      "rewards/margins": 0.5965246558189392,
+      "rewards/rejected": -1.3551405668258667,
+      "step": 285
+    },
+    {
+      "epoch": 2.16,
+      "grad_norm": 7.693954508671863,
+      "learning_rate": 1.5449438202247188e-07,
+      "logps/chosen": -48.08583068847656,
+      "logps/rejected": -53.51144027709961,
+      "loss": 0.5882,
+      "losses/dpo": 0.8339239954948425,
+      "losses/sft": 1.617476224899292,
+      "losses/total": 0.8339239954948425,
+      "ref_logps/chosen": -40.02094268798828,
+      "ref_logps/rejected": -41.421348571777344,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.8064886331558228,
+      "rewards/margins": 0.4025205969810486,
+      "rewards/rejected": -1.2090092897415161,
+      "step": 286
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 6.936991994103028,
+      "learning_rate": 1.5308988764044944e-07,
+      "logps/chosen": -42.915550231933594,
+      "logps/rejected": -54.570682525634766,
+      "loss": 0.5427,
+      "losses/dpo": 0.570111095905304,
+      "losses/sft": 1.7627439498901367,
+      "losses/total": 0.570111095905304,
+      "ref_logps/chosen": -34.83842468261719,
+      "ref_logps/rejected": -41.10365295410156,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8077125549316406,
+      "rewards/margins": 0.5389906167984009,
+      "rewards/rejected": -1.346703052520752,
+      "step": 287
+    },
+    {
+      "epoch": 2.17,
+      "grad_norm": 7.22270723761247,
+      "learning_rate": 1.5168539325842697e-07,
+      "logps/chosen": -42.23722457885742,
+      "logps/rejected": -57.404205322265625,
+      "loss": 0.529,
+      "losses/dpo": 0.5674354434013367,
+      "losses/sft": 1.5719692707061768,
+      "losses/total": 0.5674354434013367,
+      "ref_logps/chosen": -34.721920013427734,
+      "ref_logps/rejected": -44.73695373535156,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.7515305280685425,
+      "rewards/margins": 0.5151941180229187,
+      "rewards/rejected": -1.266724705696106,
+      "step": 288
+    },
+    {
+      "epoch": 2.18,
+      "grad_norm": 8.53784031336331,
+      "learning_rate": 1.502808988764045e-07,
+      "logps/chosen": -48.22527313232422,
+      "logps/rejected": -58.22871398925781,
+      "loss": 0.5799,
+      "losses/dpo": 0.5890235900878906,
+      "losses/sft": 1.6156002283096313,
+      "losses/total": 0.5890235900878906,
+      "ref_logps/chosen": -39.35710525512695,
+      "ref_logps/rejected": -45.079322814941406,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.8868170976638794,
+      "rewards/margins": 0.4281224012374878,
+      "rewards/rejected": -1.3149394989013672,
+      "step": 289
+    },
+    {
+      "epoch": 2.19,
+      "grad_norm": 6.885607484250047,
+      "learning_rate": 1.4887640449438203e-07,
+      "logps/chosen": -42.00331115722656,
+      "logps/rejected": -51.58038330078125,
+      "loss": 0.5568,
+      "losses/dpo": 0.602211058139801,
+      "losses/sft": 1.4960790872573853,
+      "losses/total": 0.602211058139801,
+      "ref_logps/chosen": -33.715057373046875,
+      "ref_logps/rejected": -39.065757751464844,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.8288247585296631,
+      "rewards/margins": 0.422637403011322,
+      "rewards/rejected": -1.2514622211456299,
+      "step": 290
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 6.966987508866502,
+      "learning_rate": 1.4747191011235953e-07,
+      "logps/chosen": -43.61931228637695,
+      "logps/rejected": -58.451629638671875,
+      "loss": 0.5594,
+      "losses/dpo": 0.5309076309204102,
+      "losses/sft": 1.636415958404541,
+      "losses/total": 0.5309076309204102,
+      "ref_logps/chosen": -34.87947463989258,
+      "ref_logps/rejected": -44.885459899902344,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.87398362159729,
+      "rewards/margins": 0.48263317346572876,
+      "rewards/rejected": -1.356616735458374,
+      "step": 291
+    },
+    {
+      "epoch": 2.2,
+      "grad_norm": 6.8192638080381744,
+      "learning_rate": 1.4606741573033706e-07,
+      "logps/chosen": -43.92414855957031,
+      "logps/rejected": -53.39807891845703,
+      "loss": 0.5452,
+      "losses/dpo": 0.6017537713050842,
+      "losses/sft": 1.7611263990402222,
+      "losses/total": 0.6017537713050842,
+      "ref_logps/chosen": -35.702823638916016,
+      "ref_logps/rejected": -40.34605407714844,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.8221321702003479,
+      "rewards/margins": 0.4830705225467682,
+      "rewards/rejected": -1.3052027225494385,
+      "step": 292
+    },
+    {
+      "epoch": 2.21,
+      "grad_norm": 8.710229032299473,
+      "learning_rate": 1.446629213483146e-07,
+      "logps/chosen": -51.8635368347168,
+      "logps/rejected": -58.34959030151367,
+      "loss": 0.601,
+      "losses/dpo": 0.5715082883834839,
+      "losses/sft": 1.490638017654419,
+      "losses/total": 0.5715082883834839,
+      "ref_logps/chosen": -42.414031982421875,
+      "ref_logps/rejected": -45.42848587036133,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.9449502229690552,
+      "rewards/margins": 0.3471601605415344,
+      "rewards/rejected": -1.2921103239059448,
+      "step": 293
+    },
+    {
+      "epoch": 2.22,
+      "grad_norm": 7.6355542087700226,
+      "learning_rate": 1.4325842696629212e-07,
+      "logps/chosen": -43.83769607543945,
+      "logps/rejected": -58.36852264404297,
+      "loss": 0.546,
+      "losses/dpo": 0.4579807221889496,
+      "losses/sft": 1.5301527976989746,
+      "losses/total": 0.4579807221889496,
+      "ref_logps/chosen": -35.81403350830078,
+      "ref_logps/rejected": -44.5776252746582,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.802366316318512,
+      "rewards/margins": 0.5767236948013306,
+      "rewards/rejected": -1.3790900707244873,
+      "step": 294
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 7.4022075570091195,
+      "learning_rate": 1.4185393258426968e-07,
+      "logps/chosen": -44.79059600830078,
+      "logps/rejected": -59.63528060913086,
+      "loss": 0.5251,
+      "losses/dpo": 0.5625388622283936,
+      "losses/sft": 1.5417966842651367,
+      "losses/total": 0.5625388622283936,
+      "ref_logps/chosen": -36.72273254394531,
+      "ref_logps/rejected": -46.061744689941406,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.806786060333252,
+      "rewards/margins": 0.5505677461624146,
+      "rewards/rejected": -1.357353925704956,
+      "step": 295
+    },
+    {
+      "epoch": 2.23,
+      "grad_norm": 7.092958234931924,
+      "learning_rate": 1.4044943820224718e-07,
+      "logps/chosen": -42.923343658447266,
+      "logps/rejected": -52.593894958496094,
+      "loss": 0.5582,
+      "losses/dpo": 0.46777036786079407,
+      "losses/sft": 1.5354235172271729,
+      "losses/total": 0.46777036786079407,
+      "ref_logps/chosen": -35.31480026245117,
+      "ref_logps/rejected": -40.524574279785156,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.7608542442321777,
+      "rewards/margins": 0.446077823638916,
+      "rewards/rejected": -1.2069320678710938,
+      "step": 296
+    },
+    {
+      "epoch": 2.24,
+      "grad_norm": 7.500648089064134,
+      "learning_rate": 1.3904494382022472e-07,
+      "logps/chosen": -43.400211334228516,
+      "logps/rejected": -54.485557556152344,
+      "loss": 0.5719,
+      "losses/dpo": 0.43427377939224243,
+      "losses/sft": 1.5346068143844604,
+      "losses/total": 0.43427377939224243,
+      "ref_logps/chosen": -35.220516204833984,
+      "ref_logps/rejected": -41.69098663330078,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.8179699182510376,
+      "rewards/margins": 0.4614875316619873,
+      "rewards/rejected": -1.2794575691223145,
+      "step": 297
+    },
+    {
+      "epoch": 2.25,
+      "grad_norm": 6.861133660639989,
+      "learning_rate": 1.3764044943820225e-07,
+      "logps/chosen": -40.74993896484375,
+      "logps/rejected": -55.73876190185547,
+      "loss": 0.5064,
+      "losses/dpo": 0.5779513716697693,
+      "losses/sft": 1.53359055519104,
+      "losses/total": 0.5779513716697693,
+      "ref_logps/chosen": -33.70279312133789,
+      "ref_logps/rejected": -42.673423767089844,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.7047147154808044,
+      "rewards/margins": 0.6018195152282715,
+      "rewards/rejected": -1.3065342903137207,
+      "step": 298
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 6.77015674340588,
+      "learning_rate": 1.3623595505617978e-07,
+      "logps/chosen": -41.57499694824219,
+      "logps/rejected": -55.820674896240234,
+      "loss": 0.5056,
+      "losses/dpo": 0.5236800909042358,
+      "losses/sft": 1.7500333786010742,
+      "losses/total": 0.5236800909042358,
+      "ref_logps/chosen": -34.20399475097656,
+      "ref_logps/rejected": -42.717681884765625,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.7371004819869995,
+      "rewards/margins": 0.5731986165046692,
+      "rewards/rejected": -1.3102991580963135,
+      "step": 299
+    },
+    {
+      "epoch": 2.26,
+      "grad_norm": 6.520455497747794,
+      "learning_rate": 1.3483146067415728e-07,
+      "logps/chosen": -40.784889221191406,
+      "logps/rejected": -53.35670471191406,
+      "loss": 0.5158,
+      "losses/dpo": 0.39004355669021606,
+      "losses/sft": 1.4663935899734497,
+      "losses/total": 0.39004355669021606,
+      "ref_logps/chosen": -33.147525787353516,
+      "ref_logps/rejected": -39.950157165527344,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.7637366056442261,
+      "rewards/margins": 0.5769186019897461,
+      "rewards/rejected": -1.3406550884246826,
+      "step": 300
+    },
+    {
+      "epoch": 2.27,
+      "grad_norm": 7.96122739673963,
+      "learning_rate": 1.334269662921348e-07,
+      "logps/chosen": -46.782169342041016,
+      "logps/rejected": -52.76530456542969,
+      "loss": 0.5923,
+      "losses/dpo": 0.6052607297897339,
+      "losses/sft": 1.6094651222229004,
+      "losses/total": 0.6052607297897339,
+      "ref_logps/chosen": -39.2327880859375,
+      "ref_logps/rejected": -41.223548889160156,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.754938006401062,
+      "rewards/margins": 0.3992377817630768,
+      "rewards/rejected": -1.1541757583618164,
+      "step": 301
+    },
+    {
+      "epoch": 2.28,
+      "grad_norm": 7.751653637126333,
+      "learning_rate": 1.3202247191011234e-07,
+      "logps/chosen": -49.38646697998047,
+      "logps/rejected": -61.543209075927734,
+      "loss": 0.5327,
+      "losses/dpo": 0.556348443031311,
+      "losses/sft": 1.8087131977081299,
+      "losses/total": 0.556348443031311,
+      "ref_logps/chosen": -40.26612091064453,
+      "ref_logps/rejected": -47.002288818359375,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.9120345115661621,
+      "rewards/margins": 0.5420576930046082,
+      "rewards/rejected": -1.454092264175415,
+      "step": 302
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 7.798526029135885,
+      "learning_rate": 1.306179775280899e-07,
+      "logps/chosen": -43.65242004394531,
+      "logps/rejected": -57.450496673583984,
+      "loss": 0.5818,
+      "losses/dpo": 0.6229327321052551,
+      "losses/sft": 1.691450834274292,
+      "losses/total": 0.6229327321052551,
+      "ref_logps/chosen": -35.82228088378906,
+      "ref_logps/rejected": -45.80632781982422,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.7830138802528381,
+      "rewards/margins": 0.3814033269882202,
+      "rewards/rejected": -1.1644171476364136,
+      "step": 303
+    },
+    {
+      "epoch": 2.29,
+      "grad_norm": 7.543077771323995,
+      "learning_rate": 1.2921348314606743e-07,
+      "logps/chosen": -44.790557861328125,
+      "logps/rejected": -61.33608627319336,
+      "loss": 0.5259,
+      "losses/dpo": 0.648471474647522,
+      "losses/sft": 1.673068881034851,
+      "losses/total": 0.648471474647522,
+      "ref_logps/chosen": -36.91156768798828,
+      "ref_logps/rejected": -48.083534240722656,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.7878991961479187,
+      "rewards/margins": 0.5373560190200806,
+      "rewards/rejected": -1.325255274772644,
+      "step": 304
+    },
+    {
+      "epoch": 2.3,
+      "grad_norm": 7.321395556756757,
+      "learning_rate": 1.2780898876404493e-07,
+      "logps/chosen": -45.843082427978516,
+      "logps/rejected": -57.27900695800781,
+      "loss": 0.5602,
+      "losses/dpo": 0.5154864192008972,
+      "losses/sft": 1.5874884128570557,
+      "losses/total": 0.5154864192008972,
+      "ref_logps/chosen": -37.167877197265625,
+      "ref_logps/rejected": -43.86834716796875,
+      "rewards/accuracies": 0.6875,
+      "rewards/chosen": -0.8675205707550049,
+      "rewards/margins": 0.4735449552536011,
+      "rewards/rejected": -1.341065526008606,
+      "step": 305
+    },
+    {
+      "epoch": 2.31,
+      "grad_norm": 7.180779900116491,
+      "learning_rate": 1.2640449438202246e-07,
+      "logps/chosen": -45.17388153076172,
+      "logps/rejected": -55.727230072021484,
+      "loss": 0.5217,
+      "losses/dpo": 0.4565548598766327,
+      "losses/sft": 1.4454078674316406,
+      "losses/total": 0.4565548598766327,
+      "ref_logps/chosen": -37.53833770751953,
+      "ref_logps/rejected": -42.80052185058594,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.7635539770126343,
+      "rewards/margins": 0.529117226600647,
+      "rewards/rejected": -1.2926712036132812,
+      "step": 306
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 7.748539788981976,
+      "learning_rate": 1.25e-07,
+      "logps/chosen": -45.61334991455078,
+      "logps/rejected": -49.54269790649414,
+      "loss": 0.5743,
+      "losses/dpo": 0.4475148916244507,
+      "losses/sft": 1.3761274814605713,
+      "losses/total": 0.4475148916244507,
+      "ref_logps/chosen": -37.51769256591797,
+      "ref_logps/rejected": -37.59453582763672,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.8095651865005493,
+      "rewards/margins": 0.38525110483169556,
+      "rewards/rejected": -1.1948162317276,
+      "step": 307
+    },
+    {
+      "epoch": 2.32,
+      "grad_norm": 7.42646045779241,
+      "learning_rate": 1.2359550561797752e-07,
+      "logps/chosen": -42.217491149902344,
+      "logps/rejected": -57.62702941894531,
+      "loss": 0.5103,
+      "losses/dpo": 0.5864957571029663,
+      "losses/sft": 1.526570439338684,
+      "losses/total": 0.5864957571029663,
+      "ref_logps/chosen": -34.16813659667969,
+      "ref_logps/rejected": -43.83499526977539,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8049358129501343,
+      "rewards/margins": 0.5742676258087158,
+      "rewards/rejected": -1.37920343875885,
+      "step": 308
+    },
+    {
+      "epoch": 2.33,
+      "grad_norm": 7.402951195988575,
+      "learning_rate": 1.2219101123595506e-07,
+      "logps/chosen": -43.753623962402344,
+      "logps/rejected": -55.725196838378906,
+      "loss": 0.5457,
+      "losses/dpo": 0.4583805501461029,
+      "losses/sft": 1.4125399589538574,
+      "losses/total": 0.4583805501461029,
+      "ref_logps/chosen": -35.22587585449219,
+      "ref_logps/rejected": -42.14164733886719,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.8527748584747314,
+      "rewards/margins": 0.5055804252624512,
+      "rewards/rejected": -1.3583552837371826,
+      "step": 309
+    },
+    {
+      "epoch": 2.34,
+      "grad_norm": 8.606023903012021,
+      "learning_rate": 1.2078651685393259e-07,
+      "logps/chosen": -52.9200439453125,
+      "logps/rejected": -61.587310791015625,
+      "loss": 0.5909,
+      "losses/dpo": 0.5809124708175659,
+      "losses/sft": 1.585126280784607,
+      "losses/total": 0.5809124708175659,
+      "ref_logps/chosen": -43.860687255859375,
+      "ref_logps/rejected": -48.097747802734375,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.9059357047080994,
+      "rewards/margins": 0.443020224571228,
+      "rewards/rejected": -1.3489558696746826,
+      "step": 310
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 7.302220410372284,
+      "learning_rate": 1.1938202247191012e-07,
+      "logps/chosen": -43.49970245361328,
+      "logps/rejected": -58.033485412597656,
+      "loss": 0.5186,
+      "losses/dpo": 0.48940473794937134,
+      "losses/sft": 1.4836596250534058,
+      "losses/total": 0.48940473794937134,
+      "ref_logps/chosen": -36.129432678222656,
+      "ref_logps/rejected": -44.55863952636719,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.7370268702507019,
+      "rewards/margins": 0.610457181930542,
+      "rewards/rejected": -1.3474839925765991,
+      "step": 311
+    },
+    {
+      "epoch": 2.35,
+      "grad_norm": 7.715132554841409,
+      "learning_rate": 1.1797752808988763e-07,
+      "logps/chosen": -45.6818733215332,
+      "logps/rejected": -57.750892639160156,
+      "loss": 0.5446,
+      "losses/dpo": 0.6576637625694275,
+      "losses/sft": 1.6136798858642578,
+      "losses/total": 0.6576637625694275,
+      "ref_logps/chosen": -37.14127731323242,
+      "ref_logps/rejected": -43.89785385131836,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.8540595769882202,
+      "rewards/margins": 0.5312445759773254,
+      "rewards/rejected": -1.3853040933609009,
+      "step": 312
+    },
+    {
+      "epoch": 2.36,
+      "grad_norm": 7.103504431576494,
+      "learning_rate": 1.1657303370786515e-07,
+      "logps/chosen": -43.971473693847656,
+      "logps/rejected": -57.290443420410156,
+      "loss": 0.5204,
+      "losses/dpo": 0.5861748456954956,
+      "losses/sft": 1.7284009456634521,
+      "losses/total": 0.5861748456954956,
+      "ref_logps/chosen": -35.535255432128906,
+      "ref_logps/rejected": -43.220550537109375,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.8436219096183777,
+      "rewards/margins": 0.5633664727210999,
+      "rewards/rejected": -1.4069883823394775,
+      "step": 313
+    },
+    {
+      "epoch": 2.37,
+      "grad_norm": 7.74539036906925,
+      "learning_rate": 1.151685393258427e-07,
+      "logps/chosen": -45.654815673828125,
+      "logps/rejected": -55.697998046875,
+      "loss": 0.5716,
+      "losses/dpo": 0.6496266722679138,
+      "losses/sft": 1.7458603382110596,
+      "losses/total": 0.6496266722679138,
+      "ref_logps/chosen": -37.42692947387695,
+      "ref_logps/rejected": -43.09944534301758,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.8227887749671936,
+      "rewards/margins": 0.43706685304641724,
+      "rewards/rejected": -1.2598556280136108,
+      "step": 314
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 7.3016087270783965,
+      "learning_rate": 1.1376404494382023e-07,
+      "logps/chosen": -44.80577087402344,
+      "logps/rejected": -58.83177185058594,
+      "loss": 0.5616,
+      "losses/dpo": 0.5281144380569458,
+      "losses/sft": 1.5373191833496094,
+      "losses/total": 0.5281144380569458,
+      "ref_logps/chosen": -35.78607940673828,
+      "ref_logps/rejected": -44.85388946533203,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.9019690155982971,
+      "rewards/margins": 0.49581989645957947,
+      "rewards/rejected": -1.3977890014648438,
+      "step": 315
+    },
+    {
+      "epoch": 2.38,
+      "grad_norm": 7.369973670185862,
+      "learning_rate": 1.1235955056179774e-07,
+      "logps/chosen": -44.48060607910156,
+      "logps/rejected": -57.20940399169922,
+      "loss": 0.5253,
+      "losses/dpo": 0.5681818723678589,
+      "losses/sft": 1.7861613035202026,
+      "losses/total": 0.5681818723678589,
+      "ref_logps/chosen": -35.98798751831055,
+      "ref_logps/rejected": -42.8939094543457,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8492615818977356,
+      "rewards/margins": 0.5822880268096924,
+      "rewards/rejected": -1.4315495491027832,
+      "step": 316
+    },
+    {
+      "epoch": 2.39,
+      "grad_norm": 7.531569588872603,
+      "learning_rate": 1.1095505617977527e-07,
+      "logps/chosen": -43.765594482421875,
+      "logps/rejected": -55.92811965942383,
+      "loss": 0.5324,
+      "losses/dpo": 0.5917935371398926,
+      "losses/sft": 1.6781896352767944,
+      "losses/total": 0.5917935371398926,
+      "ref_logps/chosen": -35.10169219970703,
+      "ref_logps/rejected": -42.32771301269531,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.8663901090621948,
+      "rewards/margins": 0.4936509132385254,
+      "rewards/rejected": -1.3600411415100098,
+      "step": 317
+    },
+    {
+      "epoch": 2.4,
+      "grad_norm": 7.00021473521157,
+      "learning_rate": 1.095505617977528e-07,
+      "logps/chosen": -43.7101936340332,
+      "logps/rejected": -55.512020111083984,
+      "loss": 0.5587,
+      "losses/dpo": 0.3991687297821045,
+      "losses/sft": 1.6147840023040771,
+      "losses/total": 0.3991687297821045,
+      "ref_logps/chosen": -34.230926513671875,
+      "ref_logps/rejected": -40.64677810668945,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.9479266405105591,
+      "rewards/margins": 0.5385974049568176,
+      "rewards/rejected": -1.4865241050720215,
+      "step": 318
+    },
+    {
+      "epoch": 2.41,
+      "grad_norm": 7.422365442434664,
+      "learning_rate": 1.0814606741573033e-07,
+      "logps/chosen": -44.69245910644531,
+      "logps/rejected": -51.95305252075195,
+      "loss": 0.5704,
+      "losses/dpo": 0.5418112277984619,
+      "losses/sft": 1.3795506954193115,
+      "losses/total": 0.5418112277984619,
+      "ref_logps/chosen": -35.97312545776367,
+      "ref_logps/rejected": -39.2027702331543,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.8719329833984375,
+      "rewards/margins": 0.40309497714042664,
+      "rewards/rejected": -1.2750279903411865,
+      "step": 319
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 7.787952340155071,
+      "learning_rate": 1.0674157303370785e-07,
+      "logps/chosen": -46.54815673828125,
+      "logps/rejected": -55.3624153137207,
+      "loss": 0.5672,
+      "losses/dpo": 0.5258245468139648,
+      "losses/sft": 1.7972207069396973,
+      "losses/total": 0.5258245468139648,
+      "ref_logps/chosen": -38.29623031616211,
+      "ref_logps/rejected": -42.49595642089844,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.8251928687095642,
+      "rewards/margins": 0.4614531695842743,
+      "rewards/rejected": -1.2866460084915161,
+      "step": 320
+    },
+    {
+      "epoch": 2.42,
+      "grad_norm": 7.716635082250954,
+      "learning_rate": 1.0533707865168538e-07,
+      "logps/chosen": -45.037723541259766,
+      "logps/rejected": -53.17112350463867,
+      "loss": 0.5781,
+      "losses/dpo": 0.7091802358627319,
+      "losses/sft": 1.6653159856796265,
+      "losses/total": 0.7091802358627319,
+      "ref_logps/chosen": -35.86992263793945,
+      "ref_logps/rejected": -39.99578857421875,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.9167801141738892,
+      "rewards/margins": 0.4007537364959717,
+      "rewards/rejected": -1.3175339698791504,
+      "step": 321
+    },
+    {
+      "epoch": 2.43,
+      "grad_norm": 6.913928315105185,
+      "learning_rate": 1.0393258426966293e-07,
+      "logps/chosen": -46.575584411621094,
+      "logps/rejected": -59.92189407348633,
+      "loss": 0.4903,
+      "losses/dpo": 0.4884983003139496,
+      "losses/sft": 1.5409932136535645,
+      "losses/total": 0.4884983003139496,
+      "ref_logps/chosen": -39.07024383544922,
+      "ref_logps/rejected": -45.98902893066406,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.750534176826477,
+      "rewards/margins": 0.6427518725395203,
+      "rewards/rejected": -1.3932859897613525,
+      "step": 322
+    },
+    {
+      "epoch": 2.44,
+      "grad_norm": 7.359357410660962,
+      "learning_rate": 1.0252808988764044e-07,
+      "logps/chosen": -43.37224578857422,
+      "logps/rejected": -57.634010314941406,
+      "loss": 0.5156,
+      "losses/dpo": 0.4852214455604553,
+      "losses/sft": 1.7198714017868042,
+      "losses/total": 0.4852214455604553,
+      "ref_logps/chosen": -34.993377685546875,
+      "ref_logps/rejected": -43.477684020996094,
+      "rewards/accuracies": 0.796875,
+      "rewards/chosen": -0.8378866910934448,
+      "rewards/margins": 0.5777460336685181,
+      "rewards/rejected": -1.415632724761963,
+      "step": 323
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 6.246986321807027,
+      "learning_rate": 1.0112359550561797e-07,
+      "logps/chosen": -39.65964889526367,
+      "logps/rejected": -53.03920364379883,
+      "loss": 0.4975,
+      "losses/dpo": 0.4038864076137543,
+      "losses/sft": 1.4372718334197998,
+      "losses/total": 0.4038864076137543,
+      "ref_logps/chosen": -32.10685348510742,
+      "ref_logps/rejected": -39.401954650878906,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.7552794218063354,
+      "rewards/margins": 0.6084451675415039,
+      "rewards/rejected": -1.3637245893478394,
+      "step": 324
+    },
+    {
+      "epoch": 2.45,
+      "grad_norm": 7.331178402164894,
+      "learning_rate": 9.971910112359549e-08,
+      "logps/chosen": -44.56993865966797,
+      "logps/rejected": -59.496734619140625,
+      "loss": 0.5273,
+      "losses/dpo": 0.4201672077178955,
+      "losses/sft": 1.5359259843826294,
+      "losses/total": 0.4201672077178955,
+      "ref_logps/chosen": -36.124656677246094,
+      "ref_logps/rejected": -45.465816497802734,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8445284962654114,
+      "rewards/margins": 0.558563232421875,
+      "rewards/rejected": -1.4030916690826416,
+      "step": 325
+    },
+    {
+      "epoch": 2.46,
+      "grad_norm": 8.06984724769854,
+      "learning_rate": 9.831460674157303e-08,
+      "logps/chosen": -48.639495849609375,
+      "logps/rejected": -56.76270294189453,
+      "loss": 0.5197,
+      "losses/dpo": 0.4748002588748932,
+      "losses/sft": 1.489260196685791,
+      "losses/total": 0.4748002588748932,
+      "ref_logps/chosen": -40.237266540527344,
+      "ref_logps/rejected": -42.580772399902344,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8402228355407715,
+      "rewards/margins": 0.5779698491096497,
+      "rewards/rejected": -1.418192744255066,
+      "step": 326
+    },
+    {
+      "epoch": 2.47,
+      "grad_norm": 7.991421566802235,
+      "learning_rate": 9.691011235955055e-08,
+      "logps/chosen": -46.860626220703125,
+      "logps/rejected": -58.88548278808594,
+      "loss": 0.5235,
+      "losses/dpo": 0.6238963603973389,
+      "losses/sft": 1.7782843112945557,
+      "losses/total": 0.6238963603973389,
+      "ref_logps/chosen": -38.90311050415039,
+      "ref_logps/rejected": -45.0700569152832,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.7957516312599182,
+      "rewards/margins": 0.5857904553413391,
+      "rewards/rejected": -1.3815419673919678,
+      "step": 327
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 7.266548128858226,
+      "learning_rate": 9.550561797752808e-08,
+      "logps/chosen": -42.59172821044922,
+      "logps/rejected": -52.10871505737305,
+      "loss": 0.5667,
+      "losses/dpo": 0.6280735731124878,
+      "losses/sft": 1.5307084321975708,
+      "losses/total": 0.6280735731124878,
+      "ref_logps/chosen": -34.418861389160156,
+      "ref_logps/rejected": -39.33207702636719,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.8172866106033325,
+      "rewards/margins": 0.46037718653678894,
+      "rewards/rejected": -1.2776637077331543,
+      "step": 328
+    },
+    {
+      "epoch": 2.48,
+      "grad_norm": 7.642236003437132,
+      "learning_rate": 9.410112359550561e-08,
+      "logps/chosen": -45.72086715698242,
+      "logps/rejected": -52.531341552734375,
+      "loss": 0.5663,
+      "losses/dpo": 0.5168911814689636,
+      "losses/sft": 1.7978273630142212,
+      "losses/total": 0.5168911814689636,
+      "ref_logps/chosen": -37.207481384277344,
+      "ref_logps/rejected": -39.487640380859375,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.851338267326355,
+      "rewards/margins": 0.4530315101146698,
+      "rewards/rejected": -1.3043696880340576,
+      "step": 329
+    },
+    {
+      "epoch": 2.49,
+      "grad_norm": 7.854817727198668,
+      "learning_rate": 9.269662921348314e-08,
+      "logps/chosen": -46.91447448730469,
+      "logps/rejected": -57.34621810913086,
+      "loss": 0.5504,
+      "losses/dpo": 0.5696989297866821,
+      "losses/sft": 1.708069086074829,
+      "losses/total": 0.5696989297866821,
+      "ref_logps/chosen": -37.96686553955078,
+      "ref_logps/rejected": -43.223960876464844,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.894761323928833,
+      "rewards/margins": 0.5174643993377686,
+      "rewards/rejected": -1.4122257232666016,
+      "step": 330
+    },
+    {
+      "epoch": 2.5,
+      "grad_norm": 7.286496155272333,
+      "learning_rate": 9.129213483146067e-08,
+      "logps/chosen": -44.248069763183594,
+      "logps/rejected": -60.32553482055664,
+      "loss": 0.5098,
+      "losses/dpo": 0.6059004664421082,
+      "losses/sft": 1.61500883102417,
+      "losses/total": 0.6059004664421082,
+      "ref_logps/chosen": -35.856903076171875,
+      "ref_logps/rejected": -46.063690185546875,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.839116632938385,
+      "rewards/margins": 0.5870682001113892,
+      "rewards/rejected": -1.426184892654419,
+      "step": 331
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 7.873630188834188,
+      "learning_rate": 8.988764044943819e-08,
+      "logps/chosen": -47.52843475341797,
+      "logps/rejected": -58.742042541503906,
+      "loss": 0.5465,
+      "losses/dpo": 0.44472765922546387,
+      "losses/sft": 1.8056182861328125,
+      "losses/total": 0.44472765922546387,
+      "ref_logps/chosen": -37.702247619628906,
+      "ref_logps/rejected": -43.76355743408203,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.982619047164917,
+      "rewards/margins": 0.5152289867401123,
+      "rewards/rejected": -1.4978480339050293,
+      "step": 332
+    },
+    {
+      "epoch": 2.51,
+      "grad_norm": 7.654569968967968,
+      "learning_rate": 8.848314606741572e-08,
+      "logps/chosen": -45.35044860839844,
+      "logps/rejected": -54.683128356933594,
+      "loss": 0.5477,
+      "losses/dpo": 0.6351585388183594,
+      "losses/sft": 1.465951681137085,
+      "losses/total": 0.6351585388183594,
+      "ref_logps/chosen": -37.354312896728516,
+      "ref_logps/rejected": -41.636627197265625,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.7996135354042053,
+      "rewards/margins": 0.505035936832428,
+      "rewards/rejected": -1.3046493530273438,
+      "step": 333
+    },
+    {
+      "epoch": 2.52,
+      "grad_norm": 7.093489997148873,
+      "learning_rate": 8.707865168539325e-08,
+      "logps/chosen": -44.989524841308594,
+      "logps/rejected": -56.54049301147461,
+      "loss": 0.5235,
+      "losses/dpo": 0.6136016845703125,
+      "losses/sft": 1.876564860343933,
+      "losses/total": 0.6136016845703125,
+      "ref_logps/chosen": -36.00385284423828,
+      "ref_logps/rejected": -41.88980484008789,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.8985673785209656,
+      "rewards/margins": 0.566501259803772,
+      "rewards/rejected": -1.4650685787200928,
+      "step": 334
+    },
+    {
+      "epoch": 2.53,
+      "grad_norm": 7.6289229336667,
+      "learning_rate": 8.567415730337078e-08,
+      "logps/chosen": -45.615413665771484,
+      "logps/rejected": -56.45619201660156,
+      "loss": 0.5586,
+      "losses/dpo": 0.5084734559059143,
+      "losses/sft": 1.6048380136489868,
+      "losses/total": 0.5084734559059143,
+      "ref_logps/chosen": -36.61585998535156,
+      "ref_logps/rejected": -42.79827880859375,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.8999553322792053,
+      "rewards/margins": 0.4658358097076416,
+      "rewards/rejected": -1.3657910823822021,
+      "step": 335
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 8.275598752517682,
+      "learning_rate": 8.426966292134831e-08,
+      "logps/chosen": -47.839508056640625,
+      "logps/rejected": -61.7794303894043,
+      "loss": 0.5368,
+      "losses/dpo": 0.5232934355735779,
+      "losses/sft": 1.4998161792755127,
+      "losses/total": 0.5232934355735779,
+      "ref_logps/chosen": -38.512969970703125,
+      "ref_logps/rejected": -46.98280334472656,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.9326539039611816,
+      "rewards/margins": 0.5470089316368103,
+      "rewards/rejected": -1.4796628952026367,
+      "step": 336
+    },
+    {
+      "epoch": 2.54,
+      "grad_norm": 6.837098147362294,
+      "learning_rate": 8.286516853932583e-08,
+      "logps/chosen": -42.03435516357422,
+      "logps/rejected": -58.38957214355469,
+      "loss": 0.4918,
+      "losses/dpo": 0.34719789028167725,
+      "losses/sft": 1.4158234596252441,
+      "losses/total": 0.34719789028167725,
+      "ref_logps/chosen": -34.547210693359375,
+      "ref_logps/rejected": -44.05992889404297,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.7487142086029053,
+      "rewards/margins": 0.6842500567436218,
+      "rewards/rejected": -1.4329640865325928,
+      "step": 337
+    },
+    {
+      "epoch": 2.55,
+      "grad_norm": 8.253144412756335,
+      "learning_rate": 8.146067415730337e-08,
+      "logps/chosen": -45.23094940185547,
+      "logps/rejected": -53.472965240478516,
+      "loss": 0.595,
+      "losses/dpo": 0.5666919350624084,
+      "losses/sft": 1.5198816061019897,
+      "losses/total": 0.5666919350624084,
+      "ref_logps/chosen": -36.06470489501953,
+      "ref_logps/rejected": -40.16011047363281,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.9166238903999329,
+      "rewards/margins": 0.4146617650985718,
+      "rewards/rejected": -1.3312857151031494,
+      "step": 338
+    },
+    {
+      "epoch": 2.56,
+      "grad_norm": 8.149761017487126,
+      "learning_rate": 8.005617977528089e-08,
+      "logps/chosen": -45.32318115234375,
+      "logps/rejected": -51.304725646972656,
+      "loss": 0.6056,
+      "losses/dpo": 0.466902494430542,
+      "losses/sft": 1.4729348421096802,
+      "losses/total": 0.466902494430542,
+      "ref_logps/chosen": -35.85576248168945,
+      "ref_logps/rejected": -37.858970642089844,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.9467417597770691,
+      "rewards/margins": 0.39783352613449097,
+      "rewards/rejected": -1.34457528591156,
+      "step": 339
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 7.054066859896987,
+      "learning_rate": 7.865168539325842e-08,
+      "logps/chosen": -45.38795471191406,
+      "logps/rejected": -57.93950653076172,
+      "loss": 0.5182,
+      "losses/dpo": 0.4842032194137573,
+      "losses/sft": 1.6942293643951416,
+      "losses/total": 0.4842032194137573,
+      "ref_logps/chosen": -36.484046936035156,
+      "ref_logps/rejected": -43.67852783203125,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.8903906345367432,
+      "rewards/margins": 0.5357075333595276,
+      "rewards/rejected": -1.426098108291626,
+      "step": 340
+    },
+    {
+      "epoch": 2.57,
+      "grad_norm": 7.258837050647915,
+      "learning_rate": 7.724719101123594e-08,
+      "logps/chosen": -46.00672149658203,
+      "logps/rejected": -59.08924865722656,
+      "loss": 0.5373,
+      "losses/dpo": 0.5424889326095581,
+      "losses/sft": 1.6475489139556885,
+      "losses/total": 0.5424889326095581,
+      "ref_logps/chosen": -37.73750305175781,
+      "ref_logps/rejected": -45.23866271972656,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.826921820640564,
+      "rewards/margins": 0.5581368207931519,
+      "rewards/rejected": -1.3850586414337158,
+      "step": 341
+    },
+    {
+      "epoch": 2.58,
+      "grad_norm": 7.988126049073018,
+      "learning_rate": 7.584269662921348e-08,
+      "logps/chosen": -46.84196472167969,
+      "logps/rejected": -55.757198333740234,
+      "loss": 0.5662,
+      "losses/dpo": 0.353384792804718,
+      "losses/sft": 1.717570424079895,
+      "losses/total": 0.353384792804718,
+      "ref_logps/chosen": -37.82433319091797,
+      "ref_logps/rejected": -42.26597213745117,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.9017627835273743,
+      "rewards/margins": 0.44735997915267944,
+      "rewards/rejected": -1.3491227626800537,
+      "step": 342
+    },
+    {
+      "epoch": 2.59,
+      "grad_norm": 7.618340962447428,
+      "learning_rate": 7.443820224719101e-08,
+      "logps/chosen": -43.753684997558594,
+      "logps/rejected": -55.105316162109375,
+      "loss": 0.546,
+      "losses/dpo": 0.6419227123260498,
+      "losses/sft": 1.6892149448394775,
+      "losses/total": 0.6419227123260498,
+      "ref_logps/chosen": -35.38850402832031,
+      "ref_logps/rejected": -41.928646087646484,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.8365182876586914,
+      "rewards/margins": 0.4811485707759857,
+      "rewards/rejected": -1.3176668882369995,
+      "step": 343
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 7.5691903171304915,
+      "learning_rate": 7.303370786516853e-08,
+      "logps/chosen": -44.20778274536133,
+      "logps/rejected": -55.97998046875,
+      "loss": 0.5407,
+      "losses/dpo": 0.5625724196434021,
+      "losses/sft": 1.5753792524337769,
+      "losses/total": 0.5625724196434021,
+      "ref_logps/chosen": -35.950294494628906,
+      "ref_logps/rejected": -42.38732147216797,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8257489204406738,
+      "rewards/margins": 0.5335172414779663,
+      "rewards/rejected": -1.3592660427093506,
+      "step": 344
+    },
+    {
+      "epoch": 2.6,
+      "grad_norm": 7.013411401019271,
+      "learning_rate": 7.162921348314606e-08,
+      "logps/chosen": -48.60981750488281,
+      "logps/rejected": -61.60570526123047,
+      "loss": 0.4779,
+      "losses/dpo": 0.5409685373306274,
+      "losses/sft": 1.6795134544372559,
+      "losses/total": 0.5409685373306274,
+      "ref_logps/chosen": -39.66438293457031,
+      "ref_logps/rejected": -45.88689422607422,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.8945437073707581,
+      "rewards/margins": 0.677337646484375,
+      "rewards/rejected": -1.5718812942504883,
+      "step": 345
+    },
+    {
+      "epoch": 2.61,
+      "grad_norm": 7.235335525204532,
+      "learning_rate": 7.022471910112359e-08,
+      "logps/chosen": -40.0158805847168,
+      "logps/rejected": -53.02748107910156,
+      "loss": 0.5321,
+      "losses/dpo": 0.5608981847763062,
+      "losses/sft": 1.2928898334503174,
+      "losses/total": 0.5608981847763062,
+      "ref_logps/chosen": -32.48070526123047,
+      "ref_logps/rejected": -40.139122009277344,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.7535171508789062,
+      "rewards/margins": 0.5353185534477234,
+      "rewards/rejected": -1.2888355255126953,
+      "step": 346
+    },
+    {
+      "epoch": 2.62,
+      "grad_norm": 7.561289554463479,
+      "learning_rate": 6.882022471910112e-08,
+      "logps/chosen": -45.8831787109375,
+      "logps/rejected": -52.74605178833008,
+      "loss": 0.5634,
+      "losses/dpo": 0.5234625935554504,
+      "losses/sft": 1.5698529481887817,
+      "losses/total": 0.5234625935554504,
+      "ref_logps/chosen": -36.858985900878906,
+      "ref_logps/rejected": -39.08251953125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9024193286895752,
+      "rewards/margins": 0.4639340043067932,
+      "rewards/rejected": -1.3663533926010132,
+      "step": 347
+    },
+    {
+      "epoch": 2.63,
+      "grad_norm": 7.714313510104845,
+      "learning_rate": 6.741573033707864e-08,
+      "logps/chosen": -47.23927307128906,
+      "logps/rejected": -56.10950469970703,
+      "loss": 0.5513,
+      "losses/dpo": 0.5834592580795288,
+      "losses/sft": 1.8191860914230347,
+      "losses/total": 0.5834592580795288,
+      "ref_logps/chosen": -38.12981414794922,
+      "ref_logps/rejected": -41.886940002441406,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.9109456539154053,
+      "rewards/margins": 0.5113106966018677,
+      "rewards/rejected": -1.422256350517273,
+      "step": 348
+    },
+    {
+      "epoch": 2.63,
+      "grad_norm": 7.158492594820948,
+      "learning_rate": 6.601123595505617e-08,
+      "logps/chosen": -48.13493347167969,
+      "logps/rejected": -60.77044677734375,
+      "loss": 0.4977,
+      "losses/dpo": 0.36096107959747314,
+      "losses/sft": 1.417677640914917,
+      "losses/total": 0.36096107959747314,
+      "ref_logps/chosen": -39.71112823486328,
+      "ref_logps/rejected": -45.55910110473633,
+      "rewards/accuracies": 0.8203125,
+      "rewards/chosen": -0.8423808813095093,
+      "rewards/margins": 0.6787533164024353,
+      "rewards/rejected": -1.5211341381072998,
+      "step": 349
+    },
+    {
+      "epoch": 2.64,
+      "grad_norm": 7.665270223156107,
+      "learning_rate": 6.460674157303371e-08,
+      "logps/chosen": -45.67338562011719,
+      "logps/rejected": -52.81538391113281,
+      "loss": 0.5593,
+      "losses/dpo": 0.5008495450019836,
+      "losses/sft": 1.4033509492874146,
+      "losses/total": 0.5008495450019836,
+      "ref_logps/chosen": -37.30530548095703,
+      "ref_logps/rejected": -39.90568542480469,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.8368085026741028,
+      "rewards/margins": 0.45416122674942017,
+      "rewards/rejected": -1.290969729423523,
+      "step": 350
+    },
+    {
+      "epoch": 2.65,
+      "grad_norm": 7.472808082581494,
+      "learning_rate": 6.320224719101123e-08,
+      "logps/chosen": -43.985435485839844,
+      "logps/rejected": -57.295692443847656,
+      "loss": 0.5187,
+      "losses/dpo": 0.5680770874023438,
+      "losses/sft": 1.4148482084274292,
+      "losses/total": 0.5680770874023438,
+      "ref_logps/chosen": -36.4133186340332,
+      "ref_logps/rejected": -43.849090576171875,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.757211446762085,
+      "rewards/margins": 0.5874490737915039,
+      "rewards/rejected": -1.3446605205535889,
+      "step": 351
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 8.189112257010201,
+      "learning_rate": 6.179775280898876e-08,
+      "logps/chosen": -47.502281188964844,
+      "logps/rejected": -54.84540939331055,
+      "loss": 0.583,
+      "losses/dpo": 0.5579338073730469,
+      "losses/sft": 1.615804672241211,
+      "losses/total": 0.5579338073730469,
+      "ref_logps/chosen": -38.33686065673828,
+      "ref_logps/rejected": -41.62626647949219,
+      "rewards/accuracies": 0.6953125,
+      "rewards/chosen": -0.9165424108505249,
+      "rewards/margins": 0.40537166595458984,
+      "rewards/rejected": -1.3219139575958252,
+      "step": 352
+    },
+    {
+      "epoch": 2.66,
+      "grad_norm": 7.805410708655585,
+      "learning_rate": 6.039325842696629e-08,
+      "logps/chosen": -44.361324310302734,
+      "logps/rejected": -59.17631149291992,
+      "loss": 0.5472,
+      "losses/dpo": 0.6015689373016357,
+      "losses/sft": 1.6676236391067505,
+      "losses/total": 0.6015689373016357,
+      "ref_logps/chosen": -35.77091979980469,
+      "ref_logps/rejected": -45.55202865600586,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.8590403199195862,
+      "rewards/margins": 0.5033884048461914,
+      "rewards/rejected": -1.362428903579712,
+      "step": 353
+    },
+    {
+      "epoch": 2.67,
+      "grad_norm": 8.028184259224918,
+      "learning_rate": 5.898876404494382e-08,
+      "logps/chosen": -46.5517463684082,
+      "logps/rejected": -56.04482650756836,
+      "loss": 0.5542,
+      "losses/dpo": 0.6546050310134888,
+      "losses/sft": 1.504585862159729,
+      "losses/total": 0.6546050310134888,
+      "ref_logps/chosen": -37.785274505615234,
+      "ref_logps/rejected": -42.51100540161133,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.8766471147537231,
+      "rewards/margins": 0.47673481702804565,
+      "rewards/rejected": -1.3533821105957031,
+      "step": 354
+    },
+    {
+      "epoch": 2.68,
+      "grad_norm": 7.260620206463691,
+      "learning_rate": 5.758426966292135e-08,
+      "logps/chosen": -48.80982971191406,
+      "logps/rejected": -56.19672393798828,
+      "loss": 0.5515,
+      "losses/dpo": 0.46307122707366943,
+      "losses/sft": 1.685928463935852,
+      "losses/total": 0.46307122707366943,
+      "ref_logps/chosen": -40.1851806640625,
+      "ref_logps/rejected": -43.02751159667969,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8624651432037354,
+      "rewards/margins": 0.4544559121131897,
+      "rewards/rejected": -1.3169212341308594,
+      "step": 355
+    },
+    {
+      "epoch": 2.69,
+      "grad_norm": 7.136349320311039,
+      "learning_rate": 5.617977528089887e-08,
+      "logps/chosen": -41.40632629394531,
+      "logps/rejected": -54.12514114379883,
+      "loss": 0.5343,
+      "losses/dpo": 0.45047110319137573,
+      "losses/sft": 1.3219261169433594,
+      "losses/total": 0.45047110319137573,
+      "ref_logps/chosen": -33.34068298339844,
+      "ref_logps/rejected": -40.50140380859375,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8065648078918457,
+      "rewards/margins": 0.555808961391449,
+      "rewards/rejected": -1.3623738288879395,
+      "step": 356
+    },
+    {
+      "epoch": 2.69,
+      "grad_norm": 7.456298216594317,
+      "learning_rate": 5.47752808988764e-08,
+      "logps/chosen": -44.206722259521484,
+      "logps/rejected": -55.71735382080078,
+      "loss": 0.5494,
+      "losses/dpo": 0.4734205901622772,
+      "losses/sft": 1.4844837188720703,
+      "losses/total": 0.4734205901622772,
+      "ref_logps/chosen": -35.47336959838867,
+      "ref_logps/rejected": -41.92726516723633,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8733350038528442,
+      "rewards/margins": 0.5056736469268799,
+      "rewards/rejected": -1.3790085315704346,
+      "step": 357
+    },
+    {
+      "epoch": 2.7,
+      "grad_norm": 7.41987426694341,
+      "learning_rate": 5.3370786516853926e-08,
+      "logps/chosen": -46.22618865966797,
+      "logps/rejected": -56.47550964355469,
+      "loss": 0.5003,
+      "losses/dpo": 0.562317430973053,
+      "losses/sft": 1.491492509841919,
+      "losses/total": 0.562317430973053,
+      "ref_logps/chosen": -37.904022216796875,
+      "ref_logps/rejected": -41.78309631347656,
+      "rewards/accuracies": 0.8046875,
+      "rewards/chosen": -0.8322172164916992,
+      "rewards/margins": 0.6370242834091187,
+      "rewards/rejected": -1.4692414999008179,
+      "step": 358
+    },
+    {
+      "epoch": 2.71,
+      "grad_norm": 6.765690296642083,
+      "learning_rate": 5.196629213483146e-08,
+      "logps/chosen": -41.32649612426758,
+      "logps/rejected": -55.117488861083984,
+      "loss": 0.4906,
+      "losses/dpo": 0.45937132835388184,
+      "losses/sft": 1.3386218547821045,
+      "losses/total": 0.45937132835388184,
+      "ref_logps/chosen": -33.87388610839844,
+      "ref_logps/rejected": -41.34483337402344,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.7452608942985535,
+      "rewards/margins": 0.6320046782493591,
+      "rewards/rejected": -1.377265453338623,
+      "step": 359
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 7.95832621655637,
+      "learning_rate": 5.056179775280899e-08,
+      "logps/chosen": -44.83673858642578,
+      "logps/rejected": -54.593666076660156,
+      "loss": 0.5529,
+      "losses/dpo": 0.5646368861198425,
+      "losses/sft": 1.3903212547302246,
+      "losses/total": 0.5646368861198425,
+      "ref_logps/chosen": -36.184173583984375,
+      "ref_logps/rejected": -41.40753173828125,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.865256667137146,
+      "rewards/margins": 0.45335638523101807,
+      "rewards/rejected": -1.318613052368164,
+      "step": 360
+    },
+    {
+      "epoch": 2.72,
+      "grad_norm": 8.359120516602266,
+      "learning_rate": 4.915730337078652e-08,
+      "logps/chosen": -48.39961624145508,
+      "logps/rejected": -54.2540397644043,
+      "loss": 0.5967,
+      "losses/dpo": 0.7173389196395874,
+      "losses/sft": 1.989745020866394,
+      "losses/total": 0.7173389196395874,
+      "ref_logps/chosen": -39.236839294433594,
+      "ref_logps/rejected": -40.71720504760742,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.9162774682044983,
+      "rewards/margins": 0.4374057650566101,
+      "rewards/rejected": -1.3536832332611084,
+      "step": 361
+    },
+    {
+      "epoch": 2.73,
+      "grad_norm": 7.8949116422203645,
+      "learning_rate": 4.775280898876404e-08,
+      "logps/chosen": -45.1904296875,
+      "logps/rejected": -55.9586181640625,
+      "loss": 0.5313,
+      "losses/dpo": 0.6307837963104248,
+      "losses/sft": 1.725508213043213,
+      "losses/total": 0.6307837963104248,
+      "ref_logps/chosen": -36.340023040771484,
+      "ref_logps/rejected": -41.507598876953125,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.8850406408309937,
+      "rewards/margins": 0.5600608587265015,
+      "rewards/rejected": -1.4451014995574951,
+      "step": 362
+    },
+    {
+      "epoch": 2.74,
+      "grad_norm": 7.439921856307659,
+      "learning_rate": 4.634831460674157e-08,
+      "logps/chosen": -47.467689514160156,
+      "logps/rejected": -55.250770568847656,
+      "loss": 0.5418,
+      "losses/dpo": 0.6079765558242798,
+      "losses/sft": 1.8188178539276123,
+      "losses/total": 0.6079765558242798,
+      "ref_logps/chosen": -38.65827560424805,
+      "ref_logps/rejected": -41.087459564208984,
+      "rewards/accuracies": 0.6796875,
+      "rewards/chosen": -0.8809411525726318,
+      "rewards/margins": 0.53538978099823,
+      "rewards/rejected": -1.4163308143615723,
+      "step": 363
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 7.250730071839225,
+      "learning_rate": 4.4943820224719096e-08,
+      "logps/chosen": -42.77532196044922,
+      "logps/rejected": -59.660240173339844,
+      "loss": 0.4573,
+      "losses/dpo": 0.4384981393814087,
+      "losses/sft": 1.4787318706512451,
+      "losses/total": 0.4384981393814087,
+      "ref_logps/chosen": -35.051734924316406,
+      "ref_logps/rejected": -44.746734619140625,
+      "rewards/accuracies": 0.8671875,
+      "rewards/chosen": -0.7723584175109863,
+      "rewards/margins": 0.7189919948577881,
+      "rewards/rejected": -1.4913504123687744,
+      "step": 364
+    },
+    {
+      "epoch": 2.75,
+      "grad_norm": 8.23044878029811,
+      "learning_rate": 4.3539325842696626e-08,
+      "logps/chosen": -49.66007995605469,
+      "logps/rejected": -60.10342025756836,
+      "loss": 0.5469,
+      "losses/dpo": 0.5084363222122192,
+      "losses/sft": 1.8791687488555908,
+      "losses/total": 0.5084363222122192,
+      "ref_logps/chosen": -40.346317291259766,
+      "ref_logps/rejected": -45.41026306152344,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9313763380050659,
+      "rewards/margins": 0.5379395484924316,
+      "rewards/rejected": -1.469315767288208,
+      "step": 365
+    },
+    {
+      "epoch": 2.76,
+      "grad_norm": 7.298326331639276,
+      "learning_rate": 4.213483146067416e-08,
+      "logps/chosen": -48.57288360595703,
+      "logps/rejected": -57.29835510253906,
+      "loss": 0.5395,
+      "losses/dpo": 0.44340649247169495,
+      "losses/sft": 1.5243843793869019,
+      "losses/total": 0.44340649247169495,
+      "ref_logps/chosen": -39.81139373779297,
+      "ref_logps/rejected": -43.033912658691406,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.8761484622955322,
+      "rewards/margins": 0.5502957701683044,
+      "rewards/rejected": -1.4264442920684814,
+      "step": 366
+    },
+    {
+      "epoch": 2.77,
+      "grad_norm": 8.51804253270616,
+      "learning_rate": 4.073033707865169e-08,
+      "logps/chosen": -44.41961669921875,
+      "logps/rejected": -53.89155578613281,
+      "loss": 0.5723,
+      "losses/dpo": 0.5301268100738525,
+      "losses/sft": 1.8131489753723145,
+      "losses/total": 0.5301268100738525,
+      "ref_logps/chosen": -35.50189971923828,
+      "ref_logps/rejected": -40.51585388183594,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8917717933654785,
+      "rewards/margins": 0.44579851627349854,
+      "rewards/rejected": -1.337570309638977,
+      "step": 367
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 7.54105725557247,
+      "learning_rate": 3.932584269662921e-08,
+      "logps/chosen": -41.12848663330078,
+      "logps/rejected": -55.290313720703125,
+      "loss": 0.54,
+      "losses/dpo": 0.49226510524749756,
+      "losses/sft": 1.37047278881073,
+      "losses/total": 0.49226510524749756,
+      "ref_logps/chosen": -32.751502990722656,
+      "ref_logps/rejected": -41.476318359375,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8376982808113098,
+      "rewards/margins": 0.5437013506889343,
+      "rewards/rejected": -1.3813996315002441,
+      "step": 368
+    },
+    {
+      "epoch": 2.78,
+      "grad_norm": 7.706900297271427,
+      "learning_rate": 3.792134831460674e-08,
+      "logps/chosen": -45.84465408325195,
+      "logps/rejected": -56.17218780517578,
+      "loss": 0.5373,
+      "losses/dpo": 0.5797220468521118,
+      "losses/sft": 1.6374412775039673,
+      "losses/total": 0.5797220468521118,
+      "ref_logps/chosen": -36.70783233642578,
+      "ref_logps/rejected": -41.77714538574219,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.913682222366333,
+      "rewards/margins": 0.5258220434188843,
+      "rewards/rejected": -1.4395041465759277,
+      "step": 369
+    },
+    {
+      "epoch": 2.79,
+      "grad_norm": 7.7547810769646555,
+      "learning_rate": 3.6516853932584266e-08,
+      "logps/chosen": -42.759098052978516,
+      "logps/rejected": -52.87897491455078,
+      "loss": 0.5836,
+      "losses/dpo": 0.7289267778396606,
+      "losses/sft": 1.7013481855392456,
+      "losses/total": 0.7289267778396606,
+      "ref_logps/chosen": -34.204769134521484,
+      "ref_logps/rejected": -40.38142776489258,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.85543292760849,
+      "rewards/margins": 0.3943214416503906,
+      "rewards/rejected": -1.2497543096542358,
+      "step": 370
+    },
+    {
+      "epoch": 2.8,
+      "grad_norm": 7.07250671481464,
+      "learning_rate": 3.5112359550561796e-08,
+      "logps/chosen": -45.09293746948242,
+      "logps/rejected": -56.41200256347656,
+      "loss": 0.5025,
+      "losses/dpo": 0.42966747283935547,
+      "losses/sft": 1.5621216297149658,
+      "losses/total": 0.42966747283935547,
+      "ref_logps/chosen": -37.37934494018555,
+      "ref_logps/rejected": -42.70643615722656,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.7713593244552612,
+      "rewards/margins": 0.599197506904602,
+      "rewards/rejected": -1.3705568313598633,
+      "step": 371
+    },
+    {
+      "epoch": 2.81,
+      "grad_norm": 8.204140371424504,
+      "learning_rate": 3.370786516853932e-08,
+      "logps/chosen": -48.024269104003906,
+      "logps/rejected": -57.5866584777832,
+      "loss": 0.5389,
+      "losses/dpo": 0.5919984579086304,
+      "losses/sft": 1.4933536052703857,
+      "losses/total": 0.5919984579086304,
+      "ref_logps/chosen": -39.3967399597168,
+      "ref_logps/rejected": -43.82619857788086,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8627532124519348,
+      "rewards/margins": 0.5132932662963867,
+      "rewards/rejected": -1.3760464191436768,
+      "step": 372
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 8.113226187403898,
+      "learning_rate": 3.230337078651686e-08,
+      "logps/chosen": -46.628257751464844,
+      "logps/rejected": -62.4483642578125,
+      "loss": 0.4997,
+      "losses/dpo": 0.4237878918647766,
+      "losses/sft": 1.5488381385803223,
+      "losses/total": 0.4237878918647766,
+      "ref_logps/chosen": -38.190887451171875,
+      "ref_logps/rejected": -47.450843811035156,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.8437370657920837,
+      "rewards/margins": 0.6560153961181641,
+      "rewards/rejected": -1.4997525215148926,
+      "step": 373
+    },
+    {
+      "epoch": 2.82,
+      "grad_norm": 7.437009897432824,
+      "learning_rate": 3.089887640449438e-08,
+      "logps/chosen": -44.36549377441406,
+      "logps/rejected": -59.504005432128906,
+      "loss": 0.4967,
+      "losses/dpo": 0.42525550723075867,
+      "losses/sft": 1.5591559410095215,
+      "losses/total": 0.42525550723075867,
+      "ref_logps/chosen": -35.93299102783203,
+      "ref_logps/rejected": -44.88318634033203,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.8432497978210449,
+      "rewards/margins": 0.6188317537307739,
+      "rewards/rejected": -1.4620814323425293,
+      "step": 374
+    },
+    {
+      "epoch": 2.83,
+      "grad_norm": 6.665463460188975,
+      "learning_rate": 2.949438202247191e-08,
+      "logps/chosen": -42.69816970825195,
+      "logps/rejected": -59.00178909301758,
+      "loss": 0.4858,
+      "losses/dpo": 0.40338996052742004,
+      "losses/sft": 1.7176090478897095,
+      "losses/total": 0.40338996052742004,
+      "ref_logps/chosen": -34.84870147705078,
+      "ref_logps/rejected": -44.43999481201172,
+      "rewards/accuracies": 0.8203125,
+      "rewards/chosen": -0.7849469184875488,
+      "rewards/margins": 0.6712321639060974,
+      "rewards/rejected": -1.456179141998291,
+      "step": 375
+    },
+    {
+      "epoch": 2.84,
+      "grad_norm": 7.288923139558504,
+      "learning_rate": 2.8089887640449436e-08,
+      "logps/chosen": -47.2071533203125,
+      "logps/rejected": -58.098148345947266,
+      "loss": 0.5076,
+      "losses/dpo": 0.5357474088668823,
+      "losses/sft": 1.654085636138916,
+      "losses/total": 0.5357474088668823,
+      "ref_logps/chosen": -38.56999588012695,
+      "ref_logps/rejected": -43.37090301513672,
+      "rewards/accuracies": 0.7734375,
+      "rewards/chosen": -0.8637155890464783,
+      "rewards/margins": 0.6090089678764343,
+      "rewards/rejected": -1.4727245569229126,
+      "step": 376
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 7.551625004814956,
+      "learning_rate": 2.6685393258426963e-08,
+      "logps/chosen": -45.72412109375,
+      "logps/rejected": -56.43421173095703,
+      "loss": 0.5474,
+      "losses/dpo": 0.544715404510498,
+      "losses/sft": 1.5618551969528198,
+      "losses/total": 0.544715404510498,
+      "ref_logps/chosen": -37.27866744995117,
+      "ref_logps/rejected": -42.84328079223633,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.8445456027984619,
+      "rewards/margins": 0.5145478248596191,
+      "rewards/rejected": -1.359093427658081,
+      "step": 377
+    },
+    {
+      "epoch": 2.85,
+      "grad_norm": 8.025864212794117,
+      "learning_rate": 2.5280898876404493e-08,
+      "logps/chosen": -45.621158599853516,
+      "logps/rejected": -60.68471145629883,
+      "loss": 0.5285,
+      "losses/dpo": 0.5538164377212524,
+      "losses/sft": 1.5718330144882202,
+      "losses/total": 0.5538164377212524,
+      "ref_logps/chosen": -36.82067108154297,
+      "ref_logps/rejected": -46.61594009399414,
+      "rewards/accuracies": 0.8125,
+      "rewards/chosen": -0.8800492286682129,
+      "rewards/margins": 0.5268282890319824,
+      "rewards/rejected": -1.4068775177001953,
+      "step": 378
+    },
+    {
+      "epoch": 2.86,
+      "grad_norm": 7.047129440366168,
+      "learning_rate": 2.387640449438202e-08,
+      "logps/chosen": -44.4951057434082,
+      "logps/rejected": -50.4869499206543,
+      "loss": 0.549,
+      "losses/dpo": 0.490747332572937,
+      "losses/sft": 1.6444151401519775,
+      "losses/total": 0.490747332572937,
+      "ref_logps/chosen": -36.696903228759766,
+      "ref_logps/rejected": -37.55984878540039,
+      "rewards/accuracies": 0.671875,
+      "rewards/chosen": -0.7798205614089966,
+      "rewards/margins": 0.5128894448280334,
+      "rewards/rejected": -1.2927099466323853,
+      "step": 379
+    },
+    {
+      "epoch": 2.87,
+      "grad_norm": 7.471899506757266,
+      "learning_rate": 2.2471910112359548e-08,
+      "logps/chosen": -47.25148391723633,
+      "logps/rejected": -59.717864990234375,
+      "loss": 0.5358,
+      "losses/dpo": 0.6100134253501892,
+      "losses/sft": 1.9196665287017822,
+      "losses/total": 0.6100134253501892,
+      "ref_logps/chosen": -37.96227264404297,
+      "ref_logps/rejected": -44.814449310302734,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.9289212226867676,
+      "rewards/margins": 0.5614204406738281,
+      "rewards/rejected": -1.4903416633605957,
+      "step": 380
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 8.004493640627455,
+      "learning_rate": 2.106741573033708e-08,
+      "logps/chosen": -44.516780853271484,
+      "logps/rejected": -51.935089111328125,
+      "loss": 0.6119,
+      "losses/dpo": 0.556452751159668,
+      "losses/sft": 1.4079639911651611,
+      "losses/total": 0.556452751159668,
+      "ref_logps/chosen": -35.908897399902344,
+      "ref_logps/rejected": -39.84672546386719,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.8607881665229797,
+      "rewards/margins": 0.34804895520210266,
+      "rewards/rejected": -1.2088370323181152,
+      "step": 381
+    },
+    {
+      "epoch": 2.88,
+      "grad_norm": 7.393197706656567,
+      "learning_rate": 1.9662921348314606e-08,
+      "logps/chosen": -44.82762908935547,
+      "logps/rejected": -59.140480041503906,
+      "loss": 0.509,
+      "losses/dpo": 0.420447438955307,
+      "losses/sft": 1.7410156726837158,
+      "losses/total": 0.420447438955307,
+      "ref_logps/chosen": -36.08792495727539,
+      "ref_logps/rejected": -43.82966613769531,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.8739705085754395,
+      "rewards/margins": 0.6571108102798462,
+      "rewards/rejected": -1.5310813188552856,
+      "step": 382
+    },
+    {
+      "epoch": 2.89,
+      "grad_norm": 7.992863219139361,
+      "learning_rate": 1.8258426966292133e-08,
+      "logps/chosen": -45.79706573486328,
+      "logps/rejected": -53.040687561035156,
+      "loss": 0.5962,
+      "losses/dpo": 0.6750953197479248,
+      "losses/sft": 1.7228975296020508,
+      "losses/total": 0.6750953197479248,
+      "ref_logps/chosen": -37.67970657348633,
+      "ref_logps/rejected": -40.953521728515625,
+      "rewards/accuracies": 0.7109375,
+      "rewards/chosen": -0.8117363452911377,
+      "rewards/margins": 0.3969798684120178,
+      "rewards/rejected": -1.2087161540985107,
+      "step": 383
+    },
+    {
+      "epoch": 2.9,
+      "grad_norm": 7.487738517511007,
+      "learning_rate": 1.685393258426966e-08,
+      "logps/chosen": -45.35090637207031,
+      "logps/rejected": -57.14335632324219,
+      "loss": 0.5354,
+      "losses/dpo": 0.5379496812820435,
+      "losses/sft": 1.6705958843231201,
+      "losses/total": 0.5379496812820435,
+      "ref_logps/chosen": -37.31737518310547,
+      "ref_logps/rejected": -43.805870056152344,
+      "rewards/accuracies": 0.7265625,
+      "rewards/chosen": -0.8033530712127686,
+      "rewards/margins": 0.5303957462310791,
+      "rewards/rejected": -1.3337488174438477,
+      "step": 384
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 8.22102430010328,
+      "learning_rate": 1.544943820224719e-08,
+      "logps/chosen": -47.1776237487793,
+      "logps/rejected": -54.27086639404297,
+      "loss": 0.5733,
+      "losses/dpo": 0.5613248348236084,
+      "losses/sft": 1.773917317390442,
+      "losses/total": 0.5613248348236084,
+      "ref_logps/chosen": -38.05524444580078,
+      "ref_logps/rejected": -40.46519470214844,
+      "rewards/accuracies": 0.6640625,
+      "rewards/chosen": -0.9122380614280701,
+      "rewards/margins": 0.4683291018009186,
+      "rewards/rejected": -1.3805670738220215,
+      "step": 385
+    },
+    {
+      "epoch": 2.91,
+      "grad_norm": 7.908569868218082,
+      "learning_rate": 1.4044943820224718e-08,
+      "logps/chosen": -44.001075744628906,
+      "logps/rejected": -60.508758544921875,
+      "loss": 0.5285,
+      "losses/dpo": 0.5084520578384399,
+      "losses/sft": 1.5907535552978516,
+      "losses/total": 0.5084520578384399,
+      "ref_logps/chosen": -34.90777587890625,
+      "ref_logps/rejected": -45.287864685058594,
+      "rewards/accuracies": 0.703125,
+      "rewards/chosen": -0.9093303084373474,
+      "rewards/margins": 0.6127593517303467,
+      "rewards/rejected": -1.5220897197723389,
+      "step": 386
+    },
+    {
+      "epoch": 2.92,
+      "grad_norm": 7.329445132361356,
+      "learning_rate": 1.2640449438202247e-08,
+      "logps/chosen": -46.82018280029297,
+      "logps/rejected": -53.613643646240234,
+      "loss": 0.521,
+      "losses/dpo": 0.5018836259841919,
+      "losses/sft": 1.6243071556091309,
+      "losses/total": 0.5018836259841919,
+      "ref_logps/chosen": -38.469993591308594,
+      "ref_logps/rejected": -39.96007537841797,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.8350194692611694,
+      "rewards/margins": 0.5303376913070679,
+      "rewards/rejected": -1.3653571605682373,
+      "step": 387
+    },
+    {
+      "epoch": 2.93,
+      "grad_norm": 7.4822766342943225,
+      "learning_rate": 1.1235955056179774e-08,
+      "logps/chosen": -46.926666259765625,
+      "logps/rejected": -55.5013427734375,
+      "loss": 0.5439,
+      "losses/dpo": 0.575495183467865,
+      "losses/sft": 1.3514134883880615,
+      "losses/total": 0.575495183467865,
+      "ref_logps/chosen": -38.2935905456543,
+      "ref_logps/rejected": -42.219791412353516,
+      "rewards/accuracies": 0.78125,
+      "rewards/chosen": -0.8633076548576355,
+      "rewards/margins": 0.46484747529029846,
+      "rewards/rejected": -1.3281550407409668,
+      "step": 388
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 7.481721913520452,
+      "learning_rate": 9.831460674157303e-09,
+      "logps/chosen": -46.69519805908203,
+      "logps/rejected": -55.18059158325195,
+      "loss": 0.5312,
+      "losses/dpo": 0.5392994284629822,
+      "losses/sft": 2.022167682647705,
+      "losses/total": 0.5392994284629822,
+      "ref_logps/chosen": -38.2025146484375,
+      "ref_logps/rejected": -41.62324905395508,
+      "rewards/accuracies": 0.7578125,
+      "rewards/chosen": -0.8492681384086609,
+      "rewards/margins": 0.506466269493103,
+      "rewards/rejected": -1.3557343482971191,
+      "step": 389
+    },
+    {
+      "epoch": 2.94,
+      "grad_norm": 8.860083156452712,
+      "learning_rate": 8.42696629213483e-09,
+      "logps/chosen": -47.0518798828125,
+      "logps/rejected": -56.05253601074219,
+      "loss": 0.6151,
+      "losses/dpo": 0.8160465955734253,
+      "losses/sft": 1.661864161491394,
+      "losses/total": 0.8160465955734253,
+      "ref_logps/chosen": -37.67930603027344,
+      "ref_logps/rejected": -42.802921295166016,
+      "rewards/accuracies": 0.6484375,
+      "rewards/chosen": -0.9372565746307373,
+      "rewards/margins": 0.3877047896385193,
+      "rewards/rejected": -1.3249614238739014,
+      "step": 390
+    },
+    {
+      "epoch": 2.95,
+      "grad_norm": 7.697893962559924,
+      "learning_rate": 7.022471910112359e-09,
+      "logps/chosen": -46.420570373535156,
+      "logps/rejected": -56.345977783203125,
+      "loss": 0.5154,
+      "losses/dpo": 0.5586492419242859,
+      "losses/sft": 1.621840476989746,
+      "losses/total": 0.5586492419242859,
+      "ref_logps/chosen": -38.219852447509766,
+      "ref_logps/rejected": -42.38871765136719,
+      "rewards/accuracies": 0.7890625,
+      "rewards/chosen": -0.8200712203979492,
+      "rewards/margins": 0.5756551027297974,
+      "rewards/rejected": -1.395726203918457,
+      "step": 391
+    },
+    {
+      "epoch": 2.96,
+      "grad_norm": 7.398815606595402,
+      "learning_rate": 5.617977528089887e-09,
+      "logps/chosen": -45.353294372558594,
+      "logps/rejected": -56.47963333129883,
+      "loss": 0.5242,
+      "losses/dpo": 0.5106035470962524,
+      "losses/sft": 1.4234966039657593,
+      "losses/total": 0.5106035470962524,
+      "ref_logps/chosen": -36.788330078125,
+      "ref_logps/rejected": -42.60810852050781,
+      "rewards/accuracies": 0.7421875,
+      "rewards/chosen": -0.8564971089363098,
+      "rewards/margins": 0.5306553244590759,
+      "rewards/rejected": -1.3871524333953857,
+      "step": 392
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 7.0449225391612895,
+      "learning_rate": 4.213483146067415e-09,
+      "logps/chosen": -44.40395736694336,
+      "logps/rejected": -53.697776794433594,
+      "loss": 0.5379,
+      "losses/dpo": 0.5200778841972351,
+      "losses/sft": 1.9024913311004639,
+      "losses/total": 0.5200778841972351,
+      "ref_logps/chosen": -36.2678337097168,
+      "ref_logps/rejected": -40.272247314453125,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.8136123418807983,
+      "rewards/margins": 0.5289404392242432,
+      "rewards/rejected": -1.3425527811050415,
+      "step": 393
+    },
+    {
+      "epoch": 2.97,
+      "grad_norm": 7.8920504953670525,
+      "learning_rate": 2.8089887640449435e-09,
+      "logps/chosen": -45.78767776489258,
+      "logps/rejected": -58.19701385498047,
+      "loss": 0.5882,
+      "losses/dpo": 0.5196930170059204,
+      "losses/sft": 1.4936178922653198,
+      "losses/total": 0.5196930170059204,
+      "ref_logps/chosen": -36.467750549316406,
+      "ref_logps/rejected": -44.57653045654297,
+      "rewards/accuracies": 0.71875,
+      "rewards/chosen": -0.9319925904273987,
+      "rewards/margins": 0.43005576729774475,
+      "rewards/rejected": -1.3620483875274658,
+      "step": 394
+    },
+    {
+      "epoch": 2.98,
+      "grad_norm": 6.749020955821219,
+      "learning_rate": 1.4044943820224717e-09,
+      "logps/chosen": -43.89699935913086,
+      "logps/rejected": -53.18260955810547,
+      "loss": 0.5308,
+      "losses/dpo": 0.7003037333488464,
+      "losses/sft": 1.696626901626587,
+      "losses/total": 0.7003037333488464,
+      "ref_logps/chosen": -35.854217529296875,
+      "ref_logps/rejected": -39.928733825683594,
+      "rewards/accuracies": 0.765625,
+      "rewards/chosen": -0.8042781949043274,
+      "rewards/margins": 0.5211097002029419,
+      "rewards/rejected": -1.3253878355026245,
+      "step": 395
+    },
+    {
+      "epoch": 2.99,
+      "grad_norm": 8.026771636400738,
+      "learning_rate": 0.0,
+      "logps/chosen": -48.76679229736328,
+      "logps/rejected": -59.84498596191406,
+      "loss": 0.5292,
+      "losses/dpo": 0.46332383155822754,
+      "losses/sft": 1.643686056137085,
+      "losses/total": 0.46332383155822754,
+      "ref_logps/chosen": -39.41192626953125,
+      "ref_logps/rejected": -44.815162658691406,
+      "rewards/accuracies": 0.734375,
+      "rewards/chosen": -0.935486912727356,
+      "rewards/margins": 0.567494809627533,
+      "rewards/rejected": -1.5029817819595337,
+      "step": 396
+    },
+    {
+      "epoch": 2.99,
+      "step": 396,
+      "total_flos": 0.0,
+      "train_loss": 0.6025580500412469,
+      "train_runtime": 11600.2001,
+      "train_samples_per_second": 4.386,
+      "train_steps_per_second": 0.034
+    }
+  ],
+  "logging_steps": 1.0,
+  "max_steps": 396,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 3,
+  "save_steps": 70,
+  "total_flos": 0.0,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}