{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.9984301412872841,
  "eval_steps": 500,
  "global_step": 159,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.006279434850863423,
      "grad_norm": 6.26599976561471,
      "learning_rate": 3.125e-08,
      "logits/chosen": 2.506397247314453,
      "logits/rejected": 2.9004628658294678,
      "logps/chosen": -304.1236877441406,
      "logps/pi_response": -206.54444885253906,
      "logps/ref_response": -206.54444885253906,
      "logps/rejected": -435.8030700683594,
      "loss": 0.6931,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "epoch": 0.06279434850863422,
      "grad_norm": 6.5208769318369715,
      "learning_rate": 3.1249999999999997e-07,
      "logits/chosen": 2.331634759902954,
      "logits/rejected": 2.5180442333221436,
      "logps/chosen": -271.9783935546875,
      "logps/pi_response": -178.7827606201172,
      "logps/ref_response": -178.90916442871094,
      "logps/rejected": -350.5011901855469,
      "loss": 0.6933,
      "rewards/accuracies": 0.4305555522441864,
      "rewards/chosen": 0.0012251290027052164,
      "rewards/margins": 0.0004269891942385584,
      "rewards/rejected": 0.0007981394301168621,
      "step": 10
    },
    {
      "epoch": 0.12558869701726844,
      "grad_norm": 6.149140437491673,
      "learning_rate": 4.990353313429303e-07,
      "logits/chosen": 2.4192910194396973,
      "logits/rejected": 2.5796756744384766,
      "logps/chosen": -264.28521728515625,
      "logps/pi_response": -169.31643676757812,
      "logps/ref_response": -172.20350646972656,
      "logps/rejected": -353.85626220703125,
      "loss": 0.6857,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": 0.020391147583723068,
      "rewards/margins": 0.01745672896504402,
      "rewards/rejected": 0.002934417687356472,
      "step": 20
    },
    {
      "epoch": 0.18838304552590268,
      "grad_norm": 5.838815655099781,
      "learning_rate": 4.882681251368548e-07,
      "logits/chosen": 2.2606046199798584,
      "logits/rejected": 2.399362087249756,
      "logps/chosen": -261.7556457519531,
      "logps/pi_response": -151.37615966796875,
      "logps/ref_response": -163.4231719970703,
      "logps/rejected": -365.0315246582031,
      "loss": 0.6617,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": 0.08448224514722824,
      "rewards/margins": 0.08167780935764313,
      "rewards/rejected": 0.002804456278681755,
      "step": 30
    },
    {
      "epoch": 0.25117739403453687,
      "grad_norm": 6.251630948090285,
      "learning_rate": 4.6604720940421207e-07,
      "logits/chosen": 2.2731361389160156,
      "logits/rejected": 2.4390079975128174,
      "logps/chosen": -283.78436279296875,
      "logps/pi_response": -162.61363220214844,
      "logps/ref_response": -179.4296417236328,
      "logps/rejected": -353.84832763671875,
      "loss": 0.6346,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.11685456335544586,
      "rewards/margins": 0.13428333401679993,
      "rewards/rejected": -0.01742876134812832,
      "step": 40
    },
    {
      "epoch": 0.3139717425431711,
      "grad_norm": 6.204506278337755,
      "learning_rate": 4.3344075855595097e-07,
      "logits/chosen": 1.9736655950546265,
      "logits/rejected": 2.127274513244629,
      "logps/chosen": -262.3057861328125,
      "logps/pi_response": -151.85891723632812,
      "logps/ref_response": -168.13865661621094,
      "logps/rejected": -346.0932922363281,
      "loss": 0.6134,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": 0.028540844097733498,
      "rewards/margins": 0.18684375286102295,
      "rewards/rejected": -0.1583028882741928,
      "step": 50
    },
    {
      "epoch": 0.37676609105180536,
      "grad_norm": 5.062174246506226,
      "learning_rate": 3.920161866827889e-07,
      "logits/chosen": 1.7557458877563477,
      "logits/rejected": 1.9249584674835205,
      "logps/chosen": -270.45989990234375,
      "logps/pi_response": -142.9950714111328,
      "logps/ref_response": -163.70513916015625,
      "logps/rejected": -408.8035583496094,
      "loss": 0.5803,
      "rewards/accuracies": 0.7562500238418579,
      "rewards/chosen": -0.011763873510062695,
      "rewards/margins": 0.3801829516887665,
      "rewards/rejected": -0.39194679260253906,
      "step": 60
    },
    {
      "epoch": 0.43956043956043955,
      "grad_norm": 4.514633956267623,
      "learning_rate": 3.4376480090239047e-07,
      "logits/chosen": 1.6569864749908447,
      "logits/rejected": 1.8295177221298218,
      "logps/chosen": -254.6250457763672,
      "logps/pi_response": -151.6630859375,
      "logps/ref_response": -169.66830444335938,
      "logps/rejected": -397.61846923828125,
      "loss": 0.5536,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.03449697047472,
      "rewards/margins": 0.402763307094574,
      "rewards/rejected": -0.437260240316391,
      "step": 70
    },
    {
      "epoch": 0.5023547880690737,
      "grad_norm": 5.51673658153614,
      "learning_rate": 2.910060778827554e-07,
      "logits/chosen": 1.7929670810699463,
      "logits/rejected": 1.9936065673828125,
      "logps/chosen": -301.0906677246094,
      "logps/pi_response": -161.8425750732422,
      "logps/ref_response": -179.93528747558594,
      "logps/rejected": -400.73992919921875,
      "loss": 0.5737,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.09138133376836777,
      "rewards/margins": 0.43019238114356995,
      "rewards/rejected": -0.5215736627578735,
      "step": 80
    },
    {
      "epoch": 0.565149136577708,
      "grad_norm": 4.60951328858753,
      "learning_rate": 2.3627616503391812e-07,
      "logits/chosen": 1.629387617111206,
      "logits/rejected": 1.7540788650512695,
      "logps/chosen": -322.956787109375,
      "logps/pi_response": -162.01455688476562,
      "logps/ref_response": -183.2974853515625,
      "logps/rejected": -443.8954162597656,
      "loss": 0.5516,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.10067585855722427,
      "rewards/margins": 0.5000637769699097,
      "rewards/rejected": -0.6007396578788757,
      "step": 90
    },
    {
      "epoch": 0.6279434850863422,
      "grad_norm": 5.912396924653729,
      "learning_rate": 1.8220596619089573e-07,
      "logits/chosen": 1.504276990890503,
      "logits/rejected": 1.6940498352050781,
      "logps/chosen": -276.9794921875,
      "logps/pi_response": -147.35867309570312,
      "logps/ref_response": -169.77037048339844,
      "logps/rejected": -378.74542236328125,
      "loss": 0.5559,
      "rewards/accuracies": 0.675000011920929,
      "rewards/chosen": -0.08432722836732864,
      "rewards/margins": 0.4150608479976654,
      "rewards/rejected": -0.4993881285190582,
      "step": 100
    },
    {
      "epoch": 0.6907378335949764,
      "grad_norm": 6.153587429768731,
      "learning_rate": 1.3139467229135998e-07,
      "logits/chosen": 1.5478519201278687,
      "logits/rejected": 1.7215379476547241,
      "logps/chosen": -282.90142822265625,
      "logps/pi_response": -164.94143676757812,
      "logps/ref_response": -186.05224609375,
      "logps/rejected": -429.9634704589844,
      "loss": 0.5423,
      "rewards/accuracies": 0.706250011920929,
      "rewards/chosen": -0.03883028030395508,
      "rewards/margins": 0.512156069278717,
      "rewards/rejected": -0.5509864091873169,
      "step": 110
    },
    {
      "epoch": 0.7535321821036107,
      "grad_norm": 5.201928644747849,
      "learning_rate": 8.628481651367875e-08,
      "logits/chosen": 1.4170118570327759,
      "logits/rejected": 1.6051431894302368,
      "logps/chosen": -289.0984802246094,
      "logps/pi_response": -153.79116821289062,
      "logps/ref_response": -173.28074645996094,
      "logps/rejected": -421.106689453125,
      "loss": 0.5484,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.0854920819401741,
      "rewards/margins": 0.5487265586853027,
      "rewards/rejected": -0.6342186331748962,
      "step": 120
    },
    {
      "epoch": 0.8163265306122449,
      "grad_norm": 5.723480822818505,
      "learning_rate": 4.904486005914027e-08,
      "logits/chosen": 1.4594230651855469,
      "logits/rejected": 1.6405918598175049,
      "logps/chosen": -275.15325927734375,
      "logps/pi_response": -150.0062713623047,
      "logps/ref_response": -171.3158721923828,
      "logps/rejected": -406.230224609375,
      "loss": 0.5302,
      "rewards/accuracies": 0.7437499761581421,
      "rewards/chosen": -0.06263308972120285,
      "rewards/margins": 0.5518540740013123,
      "rewards/rejected": -0.6144872307777405,
      "step": 130
    },
    {
      "epoch": 0.8791208791208791,
      "grad_norm": 6.184911001401629,
      "learning_rate": 2.1464952759020856e-08,
      "logits/chosen": 1.5727994441986084,
      "logits/rejected": 1.7528778314590454,
      "logps/chosen": -298.0870056152344,
      "logps/pi_response": -154.67698669433594,
      "logps/ref_response": -175.67694091796875,
      "logps/rejected": -445.009521484375,
      "loss": 0.5339,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.09744294732809067,
      "rewards/margins": 0.645055890083313,
      "rewards/rejected": -0.7424989342689514,
      "step": 140
    },
    {
      "epoch": 0.9419152276295133,
      "grad_norm": 4.863112258557619,
      "learning_rate": 4.8708793644441086e-09,
      "logits/chosen": 1.399854302406311,
      "logits/rejected": 1.5590882301330566,
      "logps/chosen": -294.4033203125,
      "logps/pi_response": -164.02032470703125,
      "logps/ref_response": -185.34927368164062,
      "logps/rejected": -422.95574951171875,
      "loss": 0.5319,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": -0.10503934323787689,
      "rewards/margins": 0.6324479579925537,
      "rewards/rejected": -0.7374873757362366,
      "step": 150
    },
    {
      "epoch": 0.9984301412872841,
      "step": 159,
      "total_flos": 0.0,
      "train_loss": 0.5863969910819575,
      "train_runtime": 3000.191,
      "train_samples_per_second": 6.792,
      "train_steps_per_second": 0.053
    }
  ],
  "logging_steps": 10,
  "max_steps": 159,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 100,
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}