{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006279434850863423,
      "grad_norm": 13.884043484189487,
      "learning_rate": 6.25e-09,
      "logits/chosen": 0.6537042260169983,
      "logits/rejected": 0.7058504819869995,
      "logps/chosen": -351.26483154296875,
      "logps/pi_response": -184.36062622070312,
      "logps/ref_response": -184.36062622070312,
      "logps/rejected": -575.6107788085938,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 15.047882494364583,
      "learning_rate": 6.25e-08,
      "logits/chosen": 0.5573610663414001,
      "logits/rejected": 0.7342559099197388,
      "logps/chosen": -347.8555908203125,
      "logps/pi_response": -139.0795440673828,
      "logps/ref_response": -139.16233825683594,
      "logps/rejected": -516.4610595703125,
      "loss": 0.6931,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -2.2081812858232297e-05,
      "rewards/margins": -0.0008097346290014684,
      "rewards/rejected": 0.0007876526215113699,
      "step": 10
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 13.801490389190754,
      "learning_rate": 9.980706626858606e-08,
      "logits/chosen": 0.6113969683647156,
      "logits/rejected": 0.8057534098625183,
      "logps/chosen": -298.69696044921875,
      "logps/pi_response": -135.04879760742188,
      "logps/ref_response": -135.14186096191406,
      "logps/rejected": -531.4634399414062,
      "loss": 0.6922,
      "rewards/accuracies": 0.5249999761581421,
      "rewards/chosen": -0.005118087865412235,
      "rewards/margins": 0.0010911704739555717,
      "rewards/rejected": -0.006209257524460554,
      "step": 20
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 12.702542077611918,
      "learning_rate": 9.765362502737097e-08,
      "logits/chosen": 0.6679919958114624,
      "logits/rejected": 0.8110311627388,
      "logps/chosen": -356.38702392578125,
      "logps/pi_response": -146.6771697998047,
      "logps/ref_response": -146.90872192382812,
      "logps/rejected": -486.9811096191406,
      "loss": 0.6884,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.009140288457274437,
      "rewards/margins": 0.007989184930920601,
      "rewards/rejected": -0.017129475250840187,
      "step": 30
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 14.943778079312272,
      "learning_rate": 9.320944188084241e-08,
      "logits/chosen": 0.49053049087524414,
      "logits/rejected": 0.7109462022781372,
      "logps/chosen": -310.234619140625,
      "logps/pi_response": -138.77902221679688,
      "logps/ref_response": -138.47763061523438,
      "logps/rejected": -532.02685546875,
      "loss": 0.6814,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.01783907599747181,
      "rewards/margins": 0.026475772261619568,
      "rewards/rejected": -0.044314850121736526,
      "step": 40
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 12.526036792984396,
      "learning_rate": 8.668815171119019e-08,
      "logits/chosen": 0.5844839811325073,
      "logits/rejected": 0.8390130996704102,
      "logps/chosen": -288.446533203125,
      "logps/pi_response": -146.24270629882812,
      "logps/ref_response": -146.1750946044922,
      "logps/rejected": -509.0594787597656,
      "loss": 0.6767,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.026793677359819412,
      "rewards/margins": 0.03173010051250458,
      "rewards/rejected": -0.05852377414703369,
      "step": 50
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 12.963675959029416,
      "learning_rate": 7.840323733655779e-08,
      "logits/chosen": 0.5787652730941772,
      "logits/rejected": 0.7449785470962524,
      "logps/chosen": -284.3918762207031,
      "logps/pi_response": -137.13790893554688,
      "logps/ref_response": -136.751953125,
      "logps/rejected": -550.5841674804688,
      "loss": 0.6653,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.05285344645380974,
      "rewards/margins": 0.0631423145532608,
      "rewards/rejected": -0.11599576473236084,
      "step": 60
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 12.831673435039468,
      "learning_rate": 6.87529601804781e-08,
      "logits/chosen": 0.39561206102371216,
      "logits/rejected": 0.6105276346206665,
      "logps/chosen": -280.70928955078125,
      "logps/pi_response": -137.5051727294922,
      "logps/ref_response": -136.8078155517578,
      "logps/rejected": -481.21539306640625,
      "loss": 0.6561,
      "rewards/accuracies": 0.6499999761581421,
      "rewards/chosen": -0.06504122167825699,
      "rewards/margins": 0.06430341303348541,
      "rewards/rejected": -0.129344642162323,
      "step": 70
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 10.720879071714686,
      "learning_rate": 5.8201215576551086e-08,
      "logits/chosen": 0.488506019115448,
      "logits/rejected": 0.6578499674797058,
      "logps/chosen": -342.3384704589844,
      "logps/pi_response": -152.0958709716797,
      "logps/ref_response": -151.61917114257812,
      "logps/rejected": -559.8889770507812,
      "loss": 0.6564,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0973019003868103,
      "rewards/margins": 0.09300337731838226,
      "rewards/rejected": -0.19030529260635376,
      "step": 80
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 12.292298954117202,
      "learning_rate": 4.725523300678362e-08,
      "logits/chosen": 0.40867680311203003,
      "logits/rejected": 0.7344620227813721,
      "logps/chosen": -293.9271240234375,
      "logps/pi_response": -148.54888916015625,
      "logps/ref_response": -147.9942169189453,
      "logps/rejected": -589.199951171875,
      "loss": 0.6488,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.0772012248635292,
      "rewards/margins": 0.13705095648765564,
      "rewards/rejected": -0.21425218880176544,
      "step": 90
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 10.50350761550398,
      "learning_rate": 3.644119323817915e-08,
      "logits/chosen": 0.6718475222587585,
      "logits/rejected": 0.8242921829223633,
      "logps/chosen": -311.2980651855469,
      "logps/pi_response": -144.72332763671875,
      "logps/ref_response": -144.91741943359375,
      "logps/rejected": -527.7688598632812,
      "loss": 0.6483,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08537373691797256,
      "rewards/margins": 0.11054044961929321,
      "rewards/rejected": -0.19591419398784637,
      "step": 100
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 10.142028760053885,
      "learning_rate": 2.6278934458271994e-08,
      "logits/chosen": 0.406488835811615,
      "logits/rejected": 0.5887733697891235,
      "logps/chosen": -326.5669250488281,
      "logps/pi_response": -145.37783813476562,
      "logps/ref_response": -144.95806884765625,
      "logps/rejected": -553.6624145507812,
      "loss": 0.6484,
      "rewards/accuracies": 0.668749988079071,
      "rewards/chosen": -0.11705747991800308,
      "rewards/margins": 0.09957389533519745,
      "rewards/rejected": -0.21663138270378113,
      "step": 110
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 12.632716685626612,
      "learning_rate": 1.725696330273575e-08,
      "logits/chosen": 0.5998940467834473,
      "logits/rejected": 0.7598485946655273,
      "logps/chosen": -308.08221435546875,
      "logps/pi_response": -143.70516967773438,
      "logps/ref_response": -143.9131317138672,
      "logps/rejected": -536.3736572265625,
      "loss": 0.6396,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.0918205976486206,
      "rewards/margins": 0.12002626806497574,
      "rewards/rejected": -0.21184685826301575,
      "step": 120
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 11.411700247352561,
      "learning_rate": 9.808972011828054e-09,
      "logits/chosen": 0.5050215721130371,
      "logits/rejected": 0.8139595985412598,
      "logps/chosen": -316.27911376953125,
      "logps/pi_response": -152.6710205078125,
      "logps/ref_response": -152.75784301757812,
      "logps/rejected": -555.2142333984375,
      "loss": 0.6455,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.09827511012554169,
      "rewards/margins": 0.11614187061786652,
      "rewards/rejected": -0.2144169807434082,
      "step": 130
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 11.032584818753614,
      "learning_rate": 4.2929905518041705e-09,
      "logits/chosen": 0.42998820543289185,
      "logits/rejected": 0.6386924982070923,
      "logps/chosen": -308.99957275390625,
      "logps/pi_response": -140.1665496826172,
      "logps/ref_response": -140.09066772460938,
      "logps/rejected": -510.5171813964844,
      "loss": 0.6427,
      "rewards/accuracies": 0.6937500238418579,
      "rewards/chosen": -0.10153601318597794,
      "rewards/margins": 0.11025869846343994,
      "rewards/rejected": -0.21179470419883728,
      "step": 140
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 9.898419952530531,
      "learning_rate": 9.741758728888216e-10,
      "logits/chosen": 0.464769184589386,
      "logits/rejected": 0.7060795426368713,
      "logps/chosen": -325.2671203613281,
      "logps/pi_response": -155.88412475585938,
      "logps/ref_response": -155.28140258789062,
      "logps/rejected": -531.9527587890625,
      "loss": 0.6455,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0997537225484848,
      "rewards/margins": 0.1331712305545807,
      "rewards/rejected": -0.23292496800422668,
      "step": 150
    },
    {
      "epoch": 0.9984301412872841,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.6602971028981719,
      "train_runtime": 3007.0775,
      "train_samples_per_second": 6.777,
      "train_steps_per_second": 0.053
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}