{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 318,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0,
      "learning_rate": 1.5625e-08,
      "logits/chosen": -2.8509469032287598,
      "logits/rejected": -2.833181858062744,
      "logps/chosen": -133.73171997070312,
      "logps/pi_response": -47.9057502746582,
      "logps/ref_response": -47.9057502746582,
      "logps/rejected": -197.21273803710938,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.03,
      "learning_rate": 1.5624999999999999e-07,
      "logits/chosen": -2.82759690284729,
      "logits/rejected": -2.798935890197754,
      "logps/chosen": -253.7941131591797,
      "logps/pi_response": -71.65193939208984,
      "logps/ref_response": -71.66093444824219,
      "logps/rejected": -215.510009765625,
      "loss": 0.693,
      "rewards/accuracies": 0.4166666567325592,
      "rewards/chosen": 2.8001426471746527e-05,
      "rewards/margins": -0.00033620299655012786,
      "rewards/rejected": 0.0003642044321168214,
      "step": 10
    },
    {
      "epoch": 0.06,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": -2.7570412158966064,
      "logits/rejected": -2.760845422744751,
      "logps/chosen": -217.1673126220703,
      "logps/pi_response": -74.27923583984375,
      "logps/ref_response": -74.31416320800781,
      "logps/rejected": -202.97946166992188,
      "loss": 0.6917,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.005049519240856171,
      "rewards/margins": 0.0024197015445679426,
      "rewards/rejected": 0.0026298172306269407,
      "step": 20
    },
    {
      "epoch": 0.09,
      "learning_rate": 4.6874999999999996e-07,
      "logits/chosen": -2.7637505531311035,
      "logits/rejected": -2.7554142475128174,
      "logps/chosen": -234.3155059814453,
      "logps/pi_response": -68.43399047851562,
      "logps/ref_response": -68.40460205078125,
      "logps/rejected": -203.23374938964844,
      "loss": 0.6859,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.028325339779257774,
      "rewards/margins": 0.01702897995710373,
      "rewards/rejected": 0.01129635889083147,
      "step": 30
    },
    {
      "epoch": 0.13,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": -2.8049890995025635,
      "logits/rejected": -2.780224084854126,
      "logps/chosen": -207.7467041015625,
      "logps/pi_response": -68.51658630371094,
      "logps/ref_response": -64.04513549804688,
      "logps/rejected": -203.04983520507812,
      "loss": 0.6743,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": 0.04143567755818367,
      "rewards/margins": 0.02963915839791298,
      "rewards/rejected": 0.011796516366302967,
      "step": 40
    },
    {
      "epoch": 0.16,
      "learning_rate": 4.951291206355559e-07,
      "logits/chosen": -2.7561471462249756,
      "logits/rejected": -2.7407054901123047,
      "logps/chosen": -256.8748474121094,
      "logps/pi_response": -92.87406921386719,
      "logps/ref_response": -74.82122039794922,
      "logps/rejected": -236.1123504638672,
      "loss": 0.6633,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": 0.0026867527049034834,
      "rewards/margins": 0.06110120937228203,
      "rewards/rejected": -0.05841444805264473,
      "step": 50
    },
    {
      "epoch": 0.19,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": -2.679486036300659,
      "logits/rejected": -2.681763172149658,
      "logps/chosen": -266.70977783203125,
      "logps/pi_response": -106.77949523925781,
      "logps/ref_response": -76.94175720214844,
      "logps/rejected": -241.783447265625,
      "loss": 0.6503,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": -0.05337841436266899,
      "rewards/margins": 0.11435987800359726,
      "rewards/rejected": -0.16773828864097595,
      "step": 60
    },
    {
      "epoch": 0.22,
      "learning_rate": 4.785350472409791e-07,
      "logits/chosen": -2.6562085151672363,
      "logits/rejected": -2.638962984085083,
      "logps/chosen": -262.48358154296875,
      "logps/pi_response": -129.61695861816406,
      "logps/ref_response": -82.19517517089844,
      "logps/rejected": -264.07568359375,
      "loss": 0.6444,
      "rewards/accuracies": 0.5687500238418579,
      "rewards/chosen": -0.20274746417999268,
      "rewards/margins": 0.09253005683422089,
      "rewards/rejected": -0.29527753591537476,
      "step": 70
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": -2.6726343631744385,
      "logits/rejected": -2.667086601257324,
      "logps/chosen": -273.891845703125,
      "logps/pi_response": -138.45387268066406,
      "logps/ref_response": -72.91886901855469,
      "logps/rejected": -269.2601318359375,
      "loss": 0.6287,
      "rewards/accuracies": 0.581250011920929,
      "rewards/chosen": -0.2682049870491028,
      "rewards/margins": 0.1169251948595047,
      "rewards/rejected": -0.3851301670074463,
      "step": 80
    },
    {
      "epoch": 0.28,
      "learning_rate": 4.5095513994085974e-07,
      "logits/chosen": -2.535644292831421,
      "logits/rejected": -2.532330274581909,
      "logps/chosen": -239.97012329101562,
      "logps/pi_response": -127.95100402832031,
      "logps/ref_response": -69.45870208740234,
      "logps/rejected": -257.403564453125,
      "loss": 0.6109,
      "rewards/accuracies": 0.6812499761581421,
      "rewards/chosen": -0.14788806438446045,
      "rewards/margins": 0.2313682734966278,
      "rewards/rejected": -0.37925636768341064,
      "step": 90
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": -2.5136990547180176,
      "logits/rejected": -2.509000062942505,
      "logps/chosen": -240.0645294189453,
      "logps/pi_response": -125.2841796875,
      "logps/ref_response": -70.416259765625,
      "logps/rejected": -254.03012084960938,
      "loss": 0.607,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.2124701738357544,
      "rewards/margins": 0.27277401089668274,
      "rewards/rejected": -0.4852442145347595,
      "step": 100
    },
    {
      "epoch": 0.35,
      "learning_rate": 4.137151834863213e-07,
      "logits/chosen": -2.480569362640381,
      "logits/rejected": -2.443432331085205,
      "logps/chosen": -275.77630615234375,
      "logps/pi_response": -136.15298461914062,
      "logps/ref_response": -71.45120239257812,
      "logps/rejected": -269.72015380859375,
      "loss": 0.6071,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.2312408983707428,
      "rewards/margins": 0.23492033779621124,
      "rewards/rejected": -0.46616125106811523,
      "step": 110
    },
    {
      "epoch": 0.38,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": -2.439021348953247,
      "logits/rejected": -2.4340755939483643,
      "logps/chosen": -268.33770751953125,
      "logps/pi_response": -167.921630859375,
      "logps/ref_response": -77.01984405517578,
      "logps/rejected": -322.13433837890625,
      "loss": 0.5893,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.35008108615875244,
      "rewards/margins": 0.42321157455444336,
      "rewards/rejected": -0.7732926607131958,
      "step": 120
    },
    {
      "epoch": 0.41,
      "learning_rate": 3.6860532770864005e-07,
      "logits/chosen": -2.347311496734619,
      "logits/rejected": -2.335073947906494,
      "logps/chosen": -315.92718505859375,
      "logps/pi_response": -209.3319091796875,
      "logps/ref_response": -67.93054962158203,
      "logps/rejected": -312.83465576171875,
      "loss": 0.5915,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.7186356782913208,
      "rewards/margins": 0.3434630036354065,
      "rewards/rejected": -1.062098741531372,
      "step": 130
    },
    {
      "epoch": 0.44,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": -2.2413113117218018,
      "logits/rejected": -2.211487293243408,
      "logps/chosen": -371.6416931152344,
      "logps/pi_response": -265.18096923828125,
      "logps/ref_response": -74.56938171386719,
      "logps/rejected": -376.59466552734375,
      "loss": 0.6044,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -1.1778886318206787,
      "rewards/margins": 0.32092222571372986,
      "rewards/rejected": -1.4988110065460205,
      "step": 140
    },
    {
      "epoch": 0.47,
      "learning_rate": 3.1779403380910425e-07,
      "logits/chosen": -2.240055561065674,
      "logits/rejected": -2.1979031562805176,
      "logps/chosen": -307.1637878417969,
      "logps/pi_response": -187.8131866455078,
      "logps/ref_response": -69.4150390625,
      "logps/rejected": -316.88873291015625,
      "loss": 0.5816,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.5614252090454102,
      "rewards/margins": 0.41102123260498047,
      "rewards/rejected": -0.9724465608596802,
      "step": 150
    },
    {
      "epoch": 0.5,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": -2.2194037437438965,
      "logits/rejected": -2.2016549110412598,
      "logps/chosen": -308.1712646484375,
      "logps/pi_response": -226.4488983154297,
      "logps/ref_response": -81.69718933105469,
      "logps/rejected": -343.4349365234375,
      "loss": 0.5699,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.6651563048362732,
      "rewards/margins": 0.4220353662967682,
      "rewards/rejected": -1.0871917009353638,
      "step": 160
    },
    {
      "epoch": 0.53,
      "learning_rate": 2.6372383496608186e-07,
      "logits/chosen": -2.233020305633545,
      "logits/rejected": -2.1675045490264893,
      "logps/chosen": -348.2539978027344,
      "logps/pi_response": -231.01455688476562,
      "logps/ref_response": -75.20973205566406,
      "logps/rejected": -334.9202880859375,
      "loss": 0.6045,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.6902152895927429,
      "rewards/margins": 0.4886610507965088,
      "rewards/rejected": -1.1788761615753174,
      "step": 170
    },
    {
      "epoch": 0.57,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": -2.137673854827881,
      "logits/rejected": -2.0975818634033203,
      "logps/chosen": -286.08172607421875,
      "logps/pi_response": -202.77284240722656,
      "logps/ref_response": -60.3970947265625,
      "logps/rejected": -297.22442626953125,
      "loss": 0.5776,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.6755498051643372,
      "rewards/margins": 0.44268402457237244,
      "rewards/rejected": -1.1182337999343872,
      "step": 180
    },
    {
      "epoch": 0.6,
      "learning_rate": 2.089939221172446e-07,
      "logits/chosen": -2.1633880138397217,
      "logits/rejected": -2.1441142559051514,
      "logps/chosen": -301.385498046875,
      "logps/pi_response": -249.9862060546875,
      "logps/ref_response": -72.61909484863281,
      "logps/rejected": -376.3998718261719,
      "loss": 0.5583,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8089518547058105,
      "rewards/margins": 0.643667995929718,
      "rewards/rejected": -1.4526197910308838,
      "step": 190
    },
    {
      "epoch": 0.63,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": -2.21587872505188,
      "logits/rejected": -2.1797242164611816,
      "logps/chosen": -383.5548400878906,
      "logps/pi_response": -243.202880859375,
      "logps/ref_response": -76.12086486816406,
      "logps/rejected": -376.36138916015625,
      "loss": 0.5584,
      "rewards/accuracies": 0.6625000238418579,
      "rewards/chosen": -0.7832227349281311,
      "rewards/margins": 0.4175417423248291,
      "rewards/rejected": -1.2007642984390259,
      "step": 200
    },
    {
      "epoch": 0.66,
      "learning_rate": 1.562351990976095e-07,
      "logits/chosen": -2.150066614151001,
      "logits/rejected": -2.121753692626953,
      "logps/chosen": -293.81488037109375,
      "logps/pi_response": -219.0130157470703,
      "logps/ref_response": -72.12835693359375,
      "logps/rejected": -308.10919189453125,
      "loss": 0.575,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.8007175326347351,
      "rewards/margins": 0.48071402311325073,
      "rewards/rejected": -1.2814315557479858,
      "step": 210
    },
    {
      "epoch": 0.69,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": -2.182744026184082,
      "logits/rejected": -2.1633565425872803,
      "logps/chosen": -320.0791931152344,
      "logps/pi_response": -229.01052856445312,
      "logps/ref_response": -76.02903747558594,
      "logps/rejected": -319.9397277832031,
      "loss": 0.5582,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.7191491723060608,
      "rewards/margins": 0.45336204767227173,
      "rewards/rejected": -1.172511339187622,
      "step": 220
    },
    {
      "epoch": 0.72,
      "learning_rate": 1.0798381331721107e-07,
      "logits/chosen": -2.094500780105591,
      "logits/rejected": -2.094238758087158,
      "logps/chosen": -340.3441467285156,
      "logps/pi_response": -242.52029418945312,
      "logps/ref_response": -78.22608184814453,
      "logps/rejected": -359.96136474609375,
      "loss": 0.5815,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.8624930381774902,
      "rewards/margins": 0.3955257534980774,
      "rewards/rejected": -1.2580187320709229,
      "step": 230
    },
    {
      "epoch": 0.75,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": -2.1416382789611816,
      "logits/rejected": -2.107877254486084,
      "logps/chosen": -334.34051513671875,
      "logps/pi_response": -253.45559692382812,
      "logps/ref_response": -71.44368743896484,
      "logps/rejected": -343.25750732421875,
      "loss": 0.567,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.9990061521530151,
      "rewards/margins": 0.39658617973327637,
      "rewards/rejected": -1.3955923318862915,
      "step": 240
    },
    {
      "epoch": 0.78,
      "learning_rate": 6.655924144404906e-08,
      "logits/chosen": -2.109405279159546,
      "logits/rejected": -2.070265769958496,
      "logps/chosen": -366.1980285644531,
      "logps/pi_response": -274.1239318847656,
      "logps/ref_response": -80.24897003173828,
      "logps/rejected": -391.06976318359375,
      "loss": 0.5633,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.0960283279418945,
      "rewards/margins": 0.4408624768257141,
      "rewards/rejected": -1.5368907451629639,
      "step": 250
    },
    {
      "epoch": 0.82,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": -2.1373915672302246,
      "logits/rejected": -2.050778865814209,
      "logps/chosen": -357.5392150878906,
      "logps/pi_response": -278.73681640625,
      "logps/ref_response": -74.540283203125,
      "logps/rejected": -366.91839599609375,
      "loss": 0.5474,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -1.0771440267562866,
      "rewards/margins": 0.7008964419364929,
      "rewards/rejected": -1.7780405282974243,
      "step": 260
    },
    {
      "epoch": 0.85,
      "learning_rate": 3.3952790595787986e-08,
      "logits/chosen": -2.031721353530884,
      "logits/rejected": -2.014037609100342,
      "logps/chosen": -339.314453125,
      "logps/pi_response": -293.8365783691406,
      "logps/ref_response": -78.09291076660156,
      "logps/rejected": -406.5647888183594,
      "loss": 0.5568,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -1.1859939098358154,
      "rewards/margins": 0.4465731084346771,
      "rewards/rejected": -1.6325668096542358,
      "step": 270
    },
    {
      "epoch": 0.88,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": -2.1228859424591064,
      "logits/rejected": -2.0986340045928955,
      "logps/chosen": -329.19244384765625,
      "logps/pi_response": -264.7164306640625,
      "logps/ref_response": -76.1480712890625,
      "logps/rejected": -405.26287841796875,
      "loss": 0.5487,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -1.001992106437683,
      "rewards/margins": 0.5500288009643555,
      "rewards/rejected": -1.552020788192749,
      "step": 280
    },
    {
      "epoch": 0.91,
      "learning_rate": 1.1731874863145142e-08,
      "logits/chosen": -2.1019539833068848,
      "logits/rejected": -2.051776885986328,
      "logps/chosen": -354.9820861816406,
      "logps/pi_response": -268.553466796875,
      "logps/ref_response": -72.30916595458984,
      "logps/rejected": -364.39324951171875,
      "loss": 0.5676,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.99445641040802,
      "rewards/margins": 0.649712324142456,
      "rewards/rejected": -1.6441688537597656,
      "step": 290
    },
    {
      "epoch": 0.94,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": -2.1944591999053955,
      "logits/rejected": -2.160240650177002,
      "logps/chosen": -352.7145080566406,
      "logps/pi_response": -258.2195129394531,
      "logps/ref_response": -73.66411590576172,
      "logps/rejected": -385.5911865234375,
      "loss": 0.5459,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.9718688726425171,
      "rewards/margins": 0.5499030351638794,
      "rewards/rejected": -1.521772027015686,
      "step": 300
    },
    {
      "epoch": 0.97,
      "learning_rate": 9.64668657069706e-10,
      "logits/chosen": -2.138836622238159,
      "logits/rejected": -2.0898382663726807,
      "logps/chosen": -320.83782958984375,
      "logps/pi_response": -251.64108276367188,
      "logps/ref_response": -73.73819732666016,
      "logps/rejected": -360.6897277832031,
      "loss": 0.5544,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.9648464322090149,
      "rewards/margins": 0.5148458480834961,
      "rewards/rejected": -1.4796922206878662,
      "step": 310
    },
    {
      "epoch": 1.0,
      "step": 318,
      "total_flos": 0.0,
      "train_loss": 0.5994115610542537,
      "train_runtime": 8245.2497,
      "train_samples_per_second": 4.943,
      "train_steps_per_second": 0.039
    }
  ],
  "logging_steps": 10,
  "max_steps": 318,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}