Upload 10 files

README.md

@@ -1,8 +1,6 @@
 ---
-library_name: transformers
-tags:
-- trl
-- orpo
+base_model: meta-llama/Meta-Llama-3-8B
+library_name: peft
 ---
 
 # Model Card for Model ID
@@ -17,7 +15,7 @@ tags:
 
 <!-- Provide a longer summary of what this model is. -->
 
-This is the model card of a 🤗 transformers model that has been pushed on the Hub. This model card has been automatically generated.
+
 
 - **Developed by:** [More Information Needed]
 - **Funded by [optional]:** [More Information Needed]
@@ -198,4 +196,7 @@ Carbon emissions can be estimated using the [Machine Learning Impact calculator]
 
 ## Model Card Contact
 
-[More Information Needed]
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab0a4d4b730f226b3662bd6e5a2eaa589e284dce0ce68739df3ad199c5deeb4e
+size 84575956

rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b3ee827a7a00012c0a116546df467feee35e70376d81a7a85b1a70eb90414d3
+size 14244

scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a0387e8589d2b776abc083193818215176f603f726060ff50a4b87d44c9b168
+size 1064

trainer_state.json

@@ -0,0 +1,1693 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9915492957746479,
+  "eval_steps": 18,
+  "global_step": 88,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.011267605633802818,
+      "grad_norm": 13.436114311218262,
+      "learning_rate": 8e-07,
+      "log_odds_chosen": 0.3692830204963684,
+      "log_odds_ratio": -0.6691731214523315,
+      "logits/chosen": -2.073309898376465,
+      "logits/rejected": -2.0528674125671387,
+      "logps/chosen": -6.679821491241455,
+      "logps/rejected": -7.049269676208496,
+      "loss": 7.3222,
+      "nll_loss": 7.255331993103027,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6679821610450745,
+      "rewards/margins": 0.03694486618041992,
+      "rewards/rejected": -0.7049269676208496,
+      "step": 1
+    },
+    {
+      "epoch": 0.022535211267605635,
+      "grad_norm": 12.660109519958496,
+      "learning_rate": 1.6e-06,
+      "log_odds_chosen": 0.05790233612060547,
+      "log_odds_ratio": -0.7999902963638306,
+      "logits/chosen": -2.122316360473633,
+      "logits/rejected": -2.1929054260253906,
+      "logps/chosen": -6.760974884033203,
+      "logps/rejected": -6.818158149719238,
+      "loss": 6.7429,
+      "nll_loss": 6.662899971008301,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6760974526405334,
+      "rewards/margins": 0.005718342959880829,
+      "rewards/rejected": -0.6818158626556396,
+      "step": 2
+    },
+    {
+      "epoch": 0.03380281690140845,
+      "grad_norm": 15.815394401550293,
+      "learning_rate": 2.4e-06,
+      "log_odds_chosen": 0.5895991325378418,
+      "log_odds_ratio": -0.5308127999305725,
+      "logits/chosen": -2.167480945587158,
+      "logits/rejected": -2.164055347442627,
+      "logps/chosen": -6.8017683029174805,
+      "logps/rejected": -7.390748977661133,
+      "loss": 7.2921,
+      "nll_loss": 7.239068031311035,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.680176854133606,
+      "rewards/margins": 0.058898091316223145,
+      "rewards/rejected": -0.7390749454498291,
+      "step": 3
+    },
+    {
+      "epoch": 0.04507042253521127,
+      "grad_norm": 15.963722229003906,
+      "learning_rate": 3.2e-06,
+      "log_odds_chosen": -0.2698395252227783,
+      "log_odds_ratio": -0.8873093128204346,
+      "logits/chosen": -2.084567070007324,
+      "logits/rejected": -2.0936269760131836,
+      "logps/chosen": -6.373527526855469,
+      "logps/rejected": -6.103606224060059,
+      "loss": 7.1739,
+      "nll_loss": 7.0851545333862305,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.637352705001831,
+      "rewards/margins": -0.026992086321115494,
+      "rewards/rejected": -0.6103606224060059,
+      "step": 4
+    },
+    {
+      "epoch": 0.056338028169014086,
+      "grad_norm": 14.874364852905273,
+      "learning_rate": 4e-06,
+      "log_odds_chosen": -0.19485323131084442,
+      "log_odds_ratio": -0.9221639633178711,
+      "logits/chosen": -2.071868896484375,
+      "logits/rejected": -2.113917827606201,
+      "logps/chosen": -6.983762741088867,
+      "logps/rejected": -6.790091037750244,
+      "loss": 7.3473,
+      "nll_loss": 7.25510835647583,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6983763575553894,
+      "rewards/margins": -0.019367169588804245,
+      "rewards/rejected": -0.6790091395378113,
+      "step": 5
+    },
+    {
+      "epoch": 0.0676056338028169,
+      "grad_norm": 12.149578094482422,
+      "learning_rate": 4.8e-06,
+      "log_odds_chosen": 1.0678657293319702,
+      "log_odds_ratio": -0.3254351019859314,
+      "logits/chosen": -2.24808669090271,
+      "logits/rejected": -2.239896774291992,
+      "logps/chosen": -6.008720397949219,
+      "logps/rejected": -7.073472023010254,
+      "loss": 6.3764,
+      "nll_loss": 6.343899250030518,
+      "rewards/accuracies": 1.0,
+      "rewards/chosen": -0.6008720993995667,
+      "rewards/margins": 0.10647513717412949,
+      "rewards/rejected": -0.7073472142219543,
+      "step": 6
+    },
+    {
+      "epoch": 0.07887323943661972,
+      "grad_norm": 17.523408889770508,
+      "learning_rate": 5.6e-06,
+      "log_odds_chosen": -0.13575804233551025,
+      "log_odds_ratio": -0.8457558751106262,
+      "logits/chosen": -2.143648386001587,
+      "logits/rejected": -2.1670777797698975,
+      "logps/chosen": -6.814686298370361,
+      "logps/rejected": -6.678918838500977,
+      "loss": 7.1252,
+      "nll_loss": 7.040646553039551,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.6814686059951782,
+      "rewards/margins": -0.01357676088809967,
+      "rewards/rejected": -0.6678918600082397,
+      "step": 7
+    },
+    {
+      "epoch": 0.09014084507042254,
+      "grad_norm": 14.829681396484375,
+      "learning_rate": 6.4e-06,
+      "log_odds_chosen": -0.01214347779750824,
+      "log_odds_ratio": -0.7939789295196533,
+      "logits/chosen": -2.1688995361328125,
+      "logits/rejected": -2.2231764793395996,
+      "logps/chosen": -6.625088691711426,
+      "logps/rejected": -6.613010406494141,
+      "loss": 7.1343,
+      "nll_loss": 7.054868221282959,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6625088453292847,
+      "rewards/margins": -0.001207888126373291,
+      "rewards/rejected": -0.6613010168075562,
+      "step": 8
+    },
+    {
+      "epoch": 0.10140845070422536,
+      "grad_norm": 13.42191219329834,
+      "learning_rate": 7.2e-06,
+      "log_odds_chosen": -0.026338055729866028,
+      "log_odds_ratio": -0.9790022969245911,
+      "logits/chosen": -2.1700892448425293,
+      "logits/rejected": -2.2076737880706787,
+      "logps/chosen": -6.157310485839844,
+      "logps/rejected": -6.1309003829956055,
+      "loss": 6.7773,
+      "nll_loss": 6.679416656494141,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.6157310605049133,
+      "rewards/margins": -0.0026410408318042755,
+      "rewards/rejected": -0.6130900382995605,
+      "step": 9
+    },
+    {
+      "epoch": 0.11267605633802817,
+      "grad_norm": 13.783522605895996,
+      "learning_rate": 8e-06,
+      "log_odds_chosen": 0.06254176795482635,
+      "log_odds_ratio": -0.7391232252120972,
+      "logits/chosen": -2.1378564834594727,
+      "logits/rejected": -2.195751667022705,
+      "logps/chosen": -6.942321300506592,
+      "logps/rejected": -7.00494384765625,
+      "loss": 7.045,
+      "nll_loss": 6.971048355102539,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.694232165813446,
+      "rewards/margins": 0.006262235343456268,
+      "rewards/rejected": -0.7004944086074829,
+      "step": 10
+    },
+    {
+      "epoch": 0.12394366197183099,
+      "grad_norm": 16.540300369262695,
+      "learning_rate": 7.897435897435898e-06,
+      "log_odds_chosen": -0.39260566234588623,
+      "log_odds_ratio": -0.9909158945083618,
+      "logits/chosen": -2.095216751098633,
+      "logits/rejected": -2.0956408977508545,
+      "logps/chosen": -6.841823101043701,
+      "logps/rejected": -6.4507737159729,
+      "loss": 7.197,
+      "nll_loss": 7.097899436950684,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.6841824054718018,
+      "rewards/margins": -0.03910496458411217,
+      "rewards/rejected": -0.6450773477554321,
+      "step": 11
+    },
+    {
+      "epoch": 0.1352112676056338,
+      "grad_norm": 13.459568977355957,
+      "learning_rate": 7.794871794871794e-06,
+      "log_odds_chosen": 0.20161187648773193,
+      "log_odds_ratio": -0.661760687828064,
+      "logits/chosen": -2.1810107231140137,
+      "logits/rejected": -2.2119851112365723,
+      "logps/chosen": -5.996310234069824,
+      "logps/rejected": -6.197461128234863,
+      "loss": 6.3979,
+      "nll_loss": 6.331758975982666,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.5996310710906982,
+      "rewards/margins": 0.020115099847316742,
+      "rewards/rejected": -0.6197460889816284,
+      "step": 12
+    },
+    {
+      "epoch": 0.14647887323943662,
+      "grad_norm": 11.37065601348877,
+      "learning_rate": 7.692307692307692e-06,
+      "log_odds_chosen": 0.2524498701095581,
+      "log_odds_ratio": -0.6112524271011353,
+      "logits/chosen": -2.2797930240631104,
+      "logits/rejected": -2.2433085441589355,
+      "logps/chosen": -5.814953327178955,
+      "logps/rejected": -6.066402435302734,
+      "loss": 6.0765,
+      "nll_loss": 6.015379428863525,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5814952850341797,
+      "rewards/margins": 0.02514498308300972,
+      "rewards/rejected": -0.6066402792930603,
+      "step": 13
+    },
+    {
+      "epoch": 0.15774647887323945,
+      "grad_norm": 12.900912284851074,
+      "learning_rate": 7.589743589743589e-06,
+      "log_odds_chosen": -0.4836970567703247,
+      "log_odds_ratio": -1.0226668119430542,
+      "logits/chosen": -2.215272903442383,
+      "logits/rejected": -2.2382960319519043,
+      "logps/chosen": -6.522824287414551,
+      "logps/rejected": -6.040271759033203,
+      "loss": 6.7429,
+      "nll_loss": 6.640633583068848,
+      "rewards/accuracies": 0.125,
+      "rewards/chosen": -0.6522824764251709,
+      "rewards/margins": -0.0482553206384182,
+      "rewards/rejected": -0.6040271520614624,
+      "step": 14
+    },
+    {
+      "epoch": 0.16901408450704225,
+      "grad_norm": 11.437642097473145,
+      "learning_rate": 7.487179487179487e-06,
+      "log_odds_chosen": 0.15005001425743103,
+      "log_odds_ratio": -0.6865947842597961,
+      "logits/chosen": -2.3359432220458984,
+      "logits/rejected": -2.382951021194458,
+      "logps/chosen": -6.031307220458984,
+      "logps/rejected": -6.181303024291992,
+      "loss": 6.4206,
+      "nll_loss": 6.3519721031188965,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.6031307578086853,
+      "rewards/margins": 0.014999527484178543,
+      "rewards/rejected": -0.6181303262710571,
+      "step": 15
+    },
+    {
+      "epoch": 0.18028169014084508,
+      "grad_norm": 11.808122634887695,
+      "learning_rate": 7.384615384615385e-06,
+      "log_odds_chosen": -0.15265953540802002,
+      "log_odds_ratio": -0.979753315448761,
+      "logits/chosen": -2.309048652648926,
+      "logits/rejected": -2.3273842334747314,
+      "logps/chosen": -6.232041358947754,
+      "logps/rejected": -6.080443859100342,
+      "loss": 6.235,
+      "nll_loss": 6.136982440948486,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.6232041120529175,
+      "rewards/margins": -0.015159666538238525,
+      "rewards/rejected": -0.608044445514679,
+      "step": 16
+    },
+    {
+      "epoch": 0.19154929577464788,
+      "grad_norm": 13.229806900024414,
+      "learning_rate": 7.282051282051281e-06,
+      "log_odds_chosen": -0.13886889815330505,
+      "log_odds_ratio": -0.8053479194641113,
+      "logits/chosen": -2.339905261993408,
+      "logits/rejected": -2.3555707931518555,
+      "logps/chosen": -5.892672061920166,
+      "logps/rejected": -5.755575180053711,
+      "loss": 6.0915,
+      "nll_loss": 6.010932445526123,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.5892672538757324,
+      "rewards/margins": -0.0137096606194973,
+      "rewards/rejected": -0.5755575895309448,
+      "step": 17
+    },
+    {
+      "epoch": 0.2028169014084507,
+      "grad_norm": 12.478164672851562,
+      "learning_rate": 7.179487179487179e-06,
+      "log_odds_chosen": -0.1651839315891266,
+      "log_odds_ratio": -0.8343566656112671,
+      "logits/chosen": -2.440002918243408,
+      "logits/rejected": -2.462096691131592,
+      "logps/chosen": -6.1440839767456055,
+      "logps/rejected": -5.97899055480957,
+      "loss": 6.1604,
+      "nll_loss": 6.076998233795166,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.6144084930419922,
+      "rewards/margins": -0.01650938391685486,
+      "rewards/rejected": -0.5978990793228149,
+      "step": 18
+    },
+    {
+      "epoch": 0.2028169014084507,
+      "eval_log_odds_chosen": 0.23532851040363312,
+      "eval_log_odds_ratio": -0.7237306833267212,
+      "eval_logits/chosen": -2.3824925422668457,
+      "eval_logits/rejected": -2.3515429496765137,
+      "eval_logps/chosen": -5.896577835083008,
+      "eval_logps/rejected": -6.131969451904297,
+      "eval_loss": 6.009965896606445,
+      "eval_nll_loss": 5.93759298324585,
+      "eval_rewards/accuracies": 0.625,
+      "eval_rewards/chosen": -0.5896577835083008,
+      "eval_rewards/margins": 0.02353917807340622,
+      "eval_rewards/rejected": -0.6131969690322876,
+      "eval_runtime": 5.3524,
+      "eval_samples_per_second": 1.495,
+      "eval_steps_per_second": 0.747,
+      "step": 18
+    },
+    {
+      "epoch": 0.2140845070422535,
+      "grad_norm": 14.38143253326416,
+      "learning_rate": 7.076923076923076e-06,
+      "log_odds_chosen": 0.41149377822875977,
+      "log_odds_ratio": -0.5679665207862854,
+      "logits/chosen": -2.2283358573913574,
+      "logits/rejected": -2.2696075439453125,
+      "logps/chosen": -5.357198715209961,
+      "logps/rejected": -5.767269611358643,
+      "loss": 5.4298,
+      "nll_loss": 5.373036861419678,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.5357198715209961,
+      "rewards/margins": 0.04100704938173294,
+      "rewards/rejected": -0.5767269730567932,
+      "step": 19
+    },
+    {
+      "epoch": 0.22535211267605634,
+      "grad_norm": 13.76131534576416,
+      "learning_rate": 6.974358974358974e-06,
+      "log_odds_chosen": -0.2197912484407425,
+      "log_odds_ratio": -0.8451388478279114,
+      "logits/chosen": -2.3156001567840576,
+      "logits/rejected": -2.3646183013916016,
+      "logps/chosen": -5.359432697296143,
+      "logps/rejected": -5.1403656005859375,
+      "loss": 5.3198,
+      "nll_loss": 5.235289096832275,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.5359432697296143,
+      "rewards/margins": -0.021906737238168716,
+      "rewards/rejected": -0.5140365362167358,
+      "step": 20
+    },
+    {
+      "epoch": 0.23661971830985915,
+      "grad_norm": 13.503231048583984,
+      "learning_rate": 6.871794871794871e-06,
+      "log_odds_chosen": -0.38248211145401,
+      "log_odds_ratio": -1.001150369644165,
+      "logits/chosen": -2.4095211029052734,
+      "logits/rejected": -2.393695592880249,
+      "logps/chosen": -5.506905555725098,
+      "logps/rejected": -5.1283159255981445,
+      "loss": 5.4066,
+      "nll_loss": 5.306459903717041,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.5506905913352966,
+      "rewards/margins": -0.037858996540308,
+      "rewards/rejected": -0.5128315687179565,
+      "step": 21
+    },
+    {
+      "epoch": 0.24788732394366197,
+      "grad_norm": 11.429216384887695,
+      "learning_rate": 6.769230769230769e-06,
+      "log_odds_chosen": 0.3094485402107239,
+      "log_odds_ratio": -0.5879026651382446,
+      "logits/chosen": -2.545229434967041,
+      "logits/rejected": -2.479078531265259,
+      "logps/chosen": -4.822021484375,
+      "logps/rejected": -5.128696918487549,
+      "loss": 5.1873,
+      "nll_loss": 5.128541469573975,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.4822021722793579,
+      "rewards/margins": 0.03066752851009369,
+      "rewards/rejected": -0.5128697156906128,
+      "step": 22
+    },
+    {
+      "epoch": 0.2591549295774648,
+      "grad_norm": 25.22710609436035,
+      "learning_rate": 6.666666666666667e-06,
+      "log_odds_chosen": -0.8079856634140015,
+      "log_odds_ratio": -1.24208664894104,
+      "logits/chosen": -2.232295036315918,
+      "logits/rejected": -2.3469386100769043,
+      "logps/chosen": -5.6021294593811035,
+      "logps/rejected": -4.799851417541504,
+      "loss": 5.6475,
+      "nll_loss": 5.523254871368408,
+      "rewards/accuracies": 0.125,
+      "rewards/chosen": -0.5602129697799683,
+      "rewards/margins": -0.08022782951593399,
+      "rewards/rejected": -0.4799851179122925,
+      "step": 23
+    },
+    {
+      "epoch": 0.2704225352112676,
+      "grad_norm": 14.158093452453613,
+      "learning_rate": 6.564102564102564e-06,
+      "log_odds_chosen": -0.18114782869815826,
+      "log_odds_ratio": -0.8821460008621216,
+      "logits/chosen": -2.4146053791046143,
+      "logits/rejected": -2.373107671737671,
+      "logps/chosen": -5.3170366287231445,
+      "logps/rejected": -5.139774799346924,
+      "loss": 5.4074,
+      "nll_loss": 5.319136619567871,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5317037105560303,
+      "rewards/margins": -0.017726212739944458,
+      "rewards/rejected": -0.5139775276184082,
+      "step": 24
+    },
+    {
+      "epoch": 0.28169014084507044,
+      "grad_norm": 16.50591468811035,
+      "learning_rate": 6.461538461538462e-06,
+      "log_odds_chosen": -0.2912433445453644,
+      "log_odds_ratio": -0.9506481885910034,
+      "logits/chosen": -2.361240863800049,
+      "logits/rejected": -2.369182586669922,
+      "logps/chosen": -5.270952224731445,
+      "logps/rejected": -4.982378959655762,
+      "loss": 5.3692,
+      "nll_loss": 5.274152755737305,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.5270952582359314,
+      "rewards/margins": -0.028857309371232986,
+      "rewards/rejected": -0.4982379674911499,
+      "step": 25
+    },
+    {
+      "epoch": 0.29295774647887324,
+      "grad_norm": 14.906917572021484,
+      "learning_rate": 6.358974358974358e-06,
+      "log_odds_chosen": -0.3039167523384094,
+      "log_odds_ratio": -0.9283076524734497,
+      "logits/chosen": -2.428100109100342,
+      "logits/rejected": -2.4127683639526367,
+      "logps/chosen": -4.490434169769287,
+      "logps/rejected": -4.19234037399292,
+      "loss": 4.7795,
+      "nll_loss": 4.686664581298828,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4490433931350708,
+      "rewards/margins": -0.02980935573577881,
+      "rewards/rejected": -0.4192340672016144,
+      "step": 26
+    },
+    {
+      "epoch": 0.30422535211267604,
+      "grad_norm": 12.076257705688477,
+      "learning_rate": 6.256410256410256e-06,
+      "log_odds_chosen": -0.006805986166000366,
+      "log_odds_ratio": -0.8016737699508667,
+      "logits/chosen": -2.520240068435669,
+      "logits/rejected": -2.4479568004608154,
+      "logps/chosen": -4.917964458465576,
+      "logps/rejected": -4.9105448722839355,
+      "loss": 4.8248,
+      "nll_loss": 4.744668483734131,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.49179643392562866,
+      "rewards/margins": -0.0007419846951961517,
+      "rewards/rejected": -0.4910544455051422,
+      "step": 27
+    },
+    {
+      "epoch": 0.3154929577464789,
+      "grad_norm": 13.027565002441406,
+      "learning_rate": 6.153846153846154e-06,
+      "log_odds_chosen": -0.18745584785938263,
+      "log_odds_ratio": -0.8756581544876099,
+      "logits/chosen": -2.5150811672210693,
+      "logits/rejected": -2.483858108520508,
+      "logps/chosen": -4.473248481750488,
+      "logps/rejected": -4.290022373199463,
+      "loss": 4.7801,
+      "nll_loss": 4.692525386810303,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.44732481241226196,
+      "rewards/margins": -0.018322546035051346,
+      "rewards/rejected": -0.4290022850036621,
+      "step": 28
+    },
+    {
+      "epoch": 0.3267605633802817,
+      "grad_norm": 11.076545715332031,
+      "learning_rate": 6.051282051282051e-06,
+      "log_odds_chosen": -0.3999772071838379,
+      "log_odds_ratio": -0.9861010909080505,
+      "logits/chosen": -2.4857661724090576,
+      "logits/rejected": -2.4484825134277344,
+      "logps/chosen": -5.028776168823242,
+      "logps/rejected": -4.632839202880859,
+      "loss": 5.1706,
+      "nll_loss": 5.071954727172852,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.5028775930404663,
+      "rewards/margins": -0.03959362953901291,
+      "rewards/rejected": -0.4632839560508728,
+      "step": 29
+    },
+    {
+      "epoch": 0.3380281690140845,
+      "grad_norm": 14.6617431640625,
+      "learning_rate": 5.948717948717948e-06,
+      "log_odds_chosen": -0.5616844892501831,
+      "log_odds_ratio": -1.1660798788070679,
+      "logits/chosen": -2.510787010192871,
+      "logits/rejected": -2.4491658210754395,
+      "logps/chosen": -4.740803241729736,
+      "logps/rejected": -4.18714714050293,
+      "loss": 4.4952,
+      "nll_loss": 4.37855339050293,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.47408032417297363,
+      "rewards/margins": -0.055365610867738724,
+      "rewards/rejected": -0.418714702129364,
+      "step": 30
+    },
+    {
+      "epoch": 0.3492957746478873,
+      "grad_norm": 12.136735916137695,
+      "learning_rate": 5.8461538461538454e-06,
+      "log_odds_chosen": -0.5842780470848083,
+      "log_odds_ratio": -1.139350175857544,
+      "logits/chosen": -2.523271083831787,
+      "logits/rejected": -2.4937047958374023,
+      "logps/chosen": -4.513538837432861,
+      "logps/rejected": -3.9382987022399902,
+      "loss": 4.4507,
+      "nll_loss": 4.336762428283691,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.45135390758514404,
+      "rewards/margins": -0.057523999363183975,
+      "rewards/rejected": -0.393829882144928,
+      "step": 31
+    },
+    {
+      "epoch": 0.36056338028169016,
+      "grad_norm": 13.099180221557617,
+      "learning_rate": 5.743589743589743e-06,
+      "log_odds_chosen": -0.37855735421180725,
+      "log_odds_ratio": -0.9612523317337036,
+      "logits/chosen": -2.5555570125579834,
+      "logits/rejected": -2.4845659732818604,
+      "logps/chosen": -4.688088417053223,
+      "logps/rejected": -4.315810680389404,
+      "loss": 4.7878,
+      "nll_loss": 4.69166374206543,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.4688088893890381,
+      "rewards/margins": -0.0372278057038784,
+      "rewards/rejected": -0.4315810799598694,
+      "step": 32
+    },
+    {
+      "epoch": 0.37183098591549296,
+      "grad_norm": 13.548160552978516,
+      "learning_rate": 5.641025641025641e-06,
+      "log_odds_chosen": -0.6679523587226868,
+      "log_odds_ratio": -1.1635067462921143,
+      "logits/chosen": -2.4850564002990723,
+      "logits/rejected": -2.4568276405334473,
+      "logps/chosen": -4.625258445739746,
+      "logps/rejected": -3.966517925262451,
+      "loss": 4.3309,
+      "nll_loss": 4.214511871337891,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4625258445739746,
+      "rewards/margins": -0.06587404012680054,
+      "rewards/rejected": -0.3966518044471741,
+      "step": 33
+    },
+    {
+      "epoch": 0.38309859154929576,
+      "grad_norm": 11.71092700958252,
+      "learning_rate": 5.538461538461538e-06,
+      "log_odds_chosen": 0.24815137684345245,
+      "log_odds_ratio": -0.642487645149231,
+      "logits/chosen": -2.511573076248169,
+      "logits/rejected": -2.466554880142212,
+      "logps/chosen": -4.356619834899902,
+      "logps/rejected": -4.603775501251221,
+      "loss": 4.2485,
+      "nll_loss": 4.18422794342041,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4356619715690613,
+      "rewards/margins": 0.024715568870306015,
+      "rewards/rejected": -0.4603775143623352,
+      "step": 34
+    },
+    {
+      "epoch": 0.39436619718309857,
+      "grad_norm": 17.99422836303711,
+      "learning_rate": 5.435897435897436e-06,
+      "log_odds_chosen": -0.4279804825782776,
+      "log_odds_ratio": -1.0195913314819336,
+      "logits/chosen": -2.4478812217712402,
+      "logits/rejected": -2.4185688495635986,
+      "logps/chosen": -4.629607200622559,
+      "logps/rejected": -4.212224960327148,
+      "loss": 4.3901,
+      "nll_loss": 4.288104057312012,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.46296072006225586,
+      "rewards/margins": -0.041738200932741165,
+      "rewards/rejected": -0.4212225377559662,
+      "step": 35
+    },
+    {
+      "epoch": 0.4056338028169014,
+      "grad_norm": 26.7180118560791,
+      "learning_rate": 5.333333333333333e-06,
+      "log_odds_chosen": -0.49325498938560486,
+      "log_odds_ratio": -1.1076762676239014,
+      "logits/chosen": -2.384472608566284,
+      "logits/rejected": -2.384478807449341,
+      "logps/chosen": -4.394374370574951,
+      "logps/rejected": -3.9007925987243652,
+      "loss": 4.5105,
+      "nll_loss": 4.399693489074707,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4394374489784241,
+      "rewards/margins": -0.04935820400714874,
+      "rewards/rejected": -0.39007923007011414,
+      "step": 36
+    },
+    {
+      "epoch": 0.4056338028169014,
+      "eval_log_odds_chosen": -0.021530766040086746,
+      "eval_log_odds_ratio": -0.7920652627944946,
+      "eval_logits/chosen": -2.437562942504883,
+      "eval_logits/rejected": -2.4226341247558594,
+      "eval_logps/chosen": -4.752222061157227,
+      "eval_logps/rejected": -4.731179237365723,
+      "eval_loss": 5.0522871017456055,
+      "eval_nll_loss": 4.973080635070801,
+      "eval_rewards/accuracies": 0.5,
+      "eval_rewards/chosen": -0.47522225975990295,
+      "eval_rewards/margins": -0.0021043792366981506,
+      "eval_rewards/rejected": -0.4731178879737854,
+      "eval_runtime": 5.3465,
+      "eval_samples_per_second": 1.496,
+      "eval_steps_per_second": 0.748,
+      "step": 36
+    },
+    {
+      "epoch": 0.4169014084507042,
+      "grad_norm": 25.756877899169922,
+      "learning_rate": 5.230769230769231e-06,
+      "log_odds_chosen": -0.04049903154373169,
+      "log_odds_ratio": -0.8640150427818298,
+      "logits/chosen": -2.536158800125122,
+      "logits/rejected": -2.481358051300049,
+      "logps/chosen": -4.724617958068848,
+      "logps/rejected": -4.686985492706299,
+      "loss": 5.0137,
+      "nll_loss": 4.927271366119385,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.4724617600440979,
+      "rewards/margins": -0.0037632212042808533,
+      "rewards/rejected": -0.46869853138923645,
+      "step": 37
+    },
+    {
+      "epoch": 0.428169014084507,
+      "grad_norm": 15.886479377746582,
+      "learning_rate": 5.128205128205129e-06,
+      "log_odds_chosen": -0.37556567788124084,
+      "log_odds_ratio": -0.9566004276275635,
+      "logits/chosen": -2.4243788719177246,
+      "logits/rejected": -2.399319887161255,
+      "logps/chosen": -4.280425071716309,
+      "logps/rejected": -3.9121856689453125,
+      "loss": 4.3575,
+      "nll_loss": 4.2618303298950195,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.428042471408844,
+      "rewards/margins": -0.03682393580675125,
+      "rewards/rejected": -0.3912185728549957,
+      "step": 38
+    },
+    {
+      "epoch": 0.4394366197183099,
+      "grad_norm": 17.65472412109375,
+      "learning_rate": 5.025641025641025e-06,
+      "log_odds_chosen": 0.2230280339717865,
+      "log_odds_ratio": -0.6618625521659851,
+      "logits/chosen": -2.3576548099517822,
+      "logits/rejected": -2.333857297897339,
+      "logps/chosen": -3.9965877532958984,
+      "logps/rejected": -4.214459419250488,
+      "loss": 4.3671,
+      "nll_loss": 4.300962448120117,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.399658739566803,
+      "rewards/margins": 0.02178717404603958,
+      "rewards/rejected": -0.42144593596458435,
+      "step": 39
+    },
+    {
+      "epoch": 0.4507042253521127,
+      "grad_norm": 11.106135368347168,
+      "learning_rate": 4.923076923076923e-06,
+      "log_odds_chosen": 0.0009728074073791504,
+      "log_odds_ratio": -0.7869582772254944,
+      "logits/chosen": -2.4159560203552246,
+      "logits/rejected": -2.4208855628967285,
+      "logps/chosen": -3.9074864387512207,
+      "logps/rejected": -3.903223991394043,
+      "loss": 4.1296,
+      "nll_loss": 4.050935745239258,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.39074867963790894,
+      "rewards/margins": -0.0004262477159500122,
+      "rewards/rejected": -0.39032241702079773,
+      "step": 40
+    },
+    {
+      "epoch": 0.4619718309859155,
+      "grad_norm": 12.57712459564209,
+      "learning_rate": 4.82051282051282e-06,
+      "log_odds_chosen": -0.2918354868888855,
+      "log_odds_ratio": -0.8778151869773865,
+      "logits/chosen": -2.3547985553741455,
+      "logits/rejected": -2.322421073913574,
+      "logps/chosen": -4.1212158203125,
+      "logps/rejected": -3.8306081295013428,
+      "loss": 4.0642,
+      "nll_loss": 3.9763693809509277,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.41212156414985657,
+      "rewards/margins": -0.029060743749141693,
+      "rewards/rejected": -0.3830608129501343,
+      "step": 41
+    },
+    {
+      "epoch": 0.4732394366197183,
+      "grad_norm": 12.822919845581055,
+      "learning_rate": 4.717948717948718e-06,
+      "log_odds_chosen": -0.810235321521759,
+      "log_odds_ratio": -1.2506808042526245,
+      "logits/chosen": -2.2839574813842773,
+      "logits/rejected": -2.288829803466797,
+      "logps/chosen": -4.443515300750732,
+      "logps/rejected": -3.658328056335449,
+      "loss": 4.3262,
+      "nll_loss": 4.201174259185791,
+      "rewards/accuracies": 0.125,
+      "rewards/chosen": -0.44435155391693115,
+      "rewards/margins": -0.07851871848106384,
+      "rewards/rejected": -0.3658328056335449,
+      "step": 42
+    },
+    {
+      "epoch": 0.48450704225352115,
+      "grad_norm": 12.537949562072754,
+      "learning_rate": 4.6153846153846145e-06,
+      "log_odds_chosen": -0.9749997854232788,
+      "log_odds_ratio": -1.3443421125411987,
+      "logits/chosen": -2.3435025215148926,
+      "logits/rejected": -2.321125030517578,
+      "logps/chosen": -4.51336145401001,
+      "logps/rejected": -3.566725254058838,
+      "loss": 4.5357,
+      "nll_loss": 4.401299476623535,
+      "rewards/accuracies": 0.125,
+      "rewards/chosen": -0.45133620500564575,
+      "rewards/margins": -0.09466366469860077,
+      "rewards/rejected": -0.3566725254058838,
+      "step": 43
+    },
+    {
+      "epoch": 0.49577464788732395,
+      "grad_norm": 10.948065757751465,
+      "learning_rate": 4.5128205128205124e-06,
+      "log_odds_chosen": -0.7950465083122253,
+      "log_odds_ratio": -1.22953200340271,
+      "logits/chosen": -2.319894790649414,
+      "logits/rejected": -2.3266992568969727,
+      "logps/chosen": -4.4190673828125,
+      "logps/rejected": -3.646463394165039,
+      "loss": 4.4846,
+      "nll_loss": 4.361649513244629,
+      "rewards/accuracies": 0.125,
+      "rewards/chosen": -0.44190675020217896,
+      "rewards/margins": -0.07726040482521057,
+      "rewards/rejected": -0.3646463453769684,
+      "step": 44
+    },
+    {
+      "epoch": 0.5070422535211268,
+      "grad_norm": 10.234577178955078,
+      "learning_rate": 4.4102564102564104e-06,
+      "log_odds_chosen": -0.03596124053001404,
+      "log_odds_ratio": -0.7954179048538208,
+      "logits/chosen": -2.3405230045318604,
+      "logits/rejected": -2.3773298263549805,
+      "logps/chosen": -3.984051465988159,
+      "logps/rejected": -3.9488115310668945,
+      "loss": 4.3053,
+      "nll_loss": 4.225733280181885,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.39840513467788696,
+      "rewards/margins": -0.003523964434862137,
+      "rewards/rejected": -0.3948811888694763,
+      "step": 45
+    },
+    {
+      "epoch": 0.5183098591549296,
+      "grad_norm": 11.955574035644531,
+      "learning_rate": 4.3076923076923076e-06,
+      "log_odds_chosen": -0.47261345386505127,
+      "log_odds_ratio": -1.0186831951141357,
+      "logits/chosen": -2.2264914512634277,
+      "logits/rejected": -2.2124805450439453,
+      "logps/chosen": -4.316086769104004,
+      "logps/rejected": -3.8509907722473145,
+      "loss": 4.2714,
+      "nll_loss": 4.169527053833008,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4316087067127228,
+      "rewards/margins": -0.046509623527526855,
+      "rewards/rejected": -0.3850990831851959,
+      "step": 46
+    },
+    {
+      "epoch": 0.5295774647887324,
+      "grad_norm": 14.009207725524902,
+      "learning_rate": 4.205128205128205e-06,
+      "log_odds_chosen": -0.5015394687652588,
+      "log_odds_ratio": -1.0697970390319824,
+      "logits/chosen": -2.216539144515991,
+      "logits/rejected": -2.236360549926758,
+      "logps/chosen": -4.524508476257324,
+      "logps/rejected": -4.033357620239258,
+      "loss": 4.2457,
+      "nll_loss": 4.138679504394531,
+      "rewards/accuracies": 0.125,
+      "rewards/chosen": -0.45245084166526794,
+      "rewards/margins": -0.04911504685878754,
+      "rewards/rejected": -0.4033357799053192,
+      "step": 47
+    },
+    {
+      "epoch": 0.5408450704225352,
+      "grad_norm": 11.495881080627441,
+      "learning_rate": 4.102564102564102e-06,
+      "log_odds_chosen": -0.0002641826868057251,
+      "log_odds_ratio": -0.7958545088768005,
+      "logits/chosen": -2.2024898529052734,
+      "logits/rejected": -2.2096405029296875,
+      "logps/chosen": -3.650709629058838,
+      "logps/rejected": -3.6475670337677,
+      "loss": 3.7327,
+      "nll_loss": 3.653069019317627,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.3650709390640259,
+      "rewards/margins": -0.0003142599016427994,
+      "rewards/rejected": -0.36475670337677,
+      "step": 48
+    },
+    {
+      "epoch": 0.5521126760563381,
+      "grad_norm": 10.599851608276367,
+      "learning_rate": 4e-06,
+      "log_odds_chosen": -0.40274348855018616,
+      "log_odds_ratio": -0.9958802461624146,
+      "logits/chosen": -2.1248817443847656,
+      "logits/rejected": -2.1678757667541504,
+      "logps/chosen": -3.9321091175079346,
+      "logps/rejected": -3.5374057292938232,
+      "loss": 3.7393,
+      "nll_loss": 3.6396658420562744,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.39321091771125793,
+      "rewards/margins": -0.03947034850716591,
+      "rewards/rejected": -0.3537406027317047,
+      "step": 49
+    },
+    {
+      "epoch": 0.5633802816901409,
+      "grad_norm": 14.113234519958496,
+      "learning_rate": 3.897435897435897e-06,
+      "log_odds_chosen": -0.15864649415016174,
+      "log_odds_ratio": -0.8118338584899902,
+      "logits/chosen": -2.182473659515381,
+      "logits/rejected": -2.2479374408721924,
+      "logps/chosen": -4.00465202331543,
+      "logps/rejected": -3.8499021530151367,
+      "loss": 3.984,
+      "nll_loss": 3.902796745300293,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4004652500152588,
+      "rewards/margins": -0.015475019812583923,
+      "rewards/rejected": -0.38499021530151367,
+      "step": 50
+    },
+    {
+      "epoch": 0.5746478873239437,
+      "grad_norm": 15.240550994873047,
+      "learning_rate": 3.7948717948717945e-06,
+      "log_odds_chosen": -0.3683556020259857,
+      "log_odds_ratio": -1.0493247509002686,
+      "logits/chosen": -2.142444133758545,
+      "logits/rejected": -2.1910910606384277,
+      "logps/chosen": -4.096914768218994,
+      "logps/rejected": -3.7346315383911133,
+      "loss": 3.9369,
+      "nll_loss": 3.8319671154022217,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4096915125846863,
+      "rewards/margins": -0.036228351294994354,
+      "rewards/rejected": -0.37346315383911133,
+      "step": 51
+    },
+    {
+      "epoch": 0.5859154929577465,
+      "grad_norm": 13.516912460327148,
+      "learning_rate": 3.6923076923076925e-06,
+      "log_odds_chosen": -0.14869263768196106,
+      "log_odds_ratio": -0.8158061504364014,
+      "logits/chosen": -2.1389424800872803,
+      "logits/rejected": -2.1591155529022217,
+      "logps/chosen": -3.914469003677368,
+      "logps/rejected": -3.7656819820404053,
+      "loss": 3.6203,
+      "nll_loss": 3.538712978363037,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.39144688844680786,
+      "rewards/margins": -0.014878695830702782,
+      "rewards/rejected": -0.3765681982040405,
+      "step": 52
+    },
+    {
+      "epoch": 0.5971830985915493,
+      "grad_norm": 12.95617961883545,
+      "learning_rate": 3.5897435897435896e-06,
+      "log_odds_chosen": -0.24592818319797516,
+      "log_odds_ratio": -0.9381157159805298,
+      "logits/chosen": -2.0287747383117676,
+      "logits/rejected": -2.1300435066223145,
+      "logps/chosen": -3.8074216842651367,
+      "logps/rejected": -3.571408748626709,
+      "loss": 3.6729,
+      "nll_loss": 3.5790419578552246,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.3807421326637268,
+      "rewards/margins": -0.023601267486810684,
+      "rewards/rejected": -0.3571408987045288,
+      "step": 53
+    },
+    {
+      "epoch": 0.6084507042253521,
+      "grad_norm": 11.833647727966309,
+      "learning_rate": 3.487179487179487e-06,
+      "log_odds_chosen": -0.5012875199317932,
+      "log_odds_ratio": -1.0330029726028442,
+      "logits/chosen": -2.141413688659668,
+      "logits/rejected": -2.1845943927764893,
+      "logps/chosen": -4.043262958526611,
+      "logps/rejected": -3.5469398498535156,
+      "loss": 3.9015,
+      "nll_loss": 3.798215627670288,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.40432626008987427,
+      "rewards/margins": -0.049632273614406586,
+      "rewards/rejected": -0.3546940088272095,
+      "step": 54
+    },
+    {
+      "epoch": 0.6084507042253521,
+      "eval_log_odds_chosen": -0.0070199985057115555,
+      "eval_log_odds_ratio": -0.8004560470581055,
+      "eval_logits/chosen": -2.2082390785217285,
+      "eval_logits/rejected": -2.2092061042785645,
+      "eval_logps/chosen": -4.265551567077637,
+      "eval_logps/rejected": -4.259222030639648,
+      "eval_loss": 4.50450325012207,
+      "eval_nll_loss": 4.42445707321167,
+      "eval_rewards/accuracies": 0.5,
+      "eval_rewards/chosen": -0.42655515670776367,
+      "eval_rewards/margins": -0.0006329789757728577,
+      "eval_rewards/rejected": -0.4259222149848938,
+      "eval_runtime": 5.3539,
+      "eval_samples_per_second": 1.494,
+      "eval_steps_per_second": 0.747,
+      "step": 54
+    },
+    {
+      "epoch": 0.6197183098591549,
+      "grad_norm": 13.9421968460083,
+      "learning_rate": 3.3846153846153843e-06,
+      "log_odds_chosen": -0.17257481813430786,
+      "log_odds_ratio": -0.8791506886482239,
+      "logits/chosen": -2.223339080810547,
+      "logits/rejected": -2.183260440826416,
+      "logps/chosen": -4.611689567565918,
+      "logps/rejected": -4.4401044845581055,
+      "loss": 4.3277,
+      "nll_loss": 4.239750862121582,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.46116903424263,
+      "rewards/margins": -0.017158538103103638,
+      "rewards/rejected": -0.444010466337204,
+      "step": 55
+    },
+    {
+      "epoch": 0.6309859154929578,
+      "grad_norm": 11.743844032287598,
+      "learning_rate": 3.282051282051282e-06,
+      "log_odds_chosen": -0.662411630153656,
+      "log_odds_ratio": -1.2490286827087402,
+      "logits/chosen": -1.9998000860214233,
+      "logits/rejected": -2.092249631881714,
+      "logps/chosen": -3.955559730529785,
+      "logps/rejected": -3.31760311126709,
+      "loss": 3.5195,
+      "nll_loss": 3.394559383392334,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.3955560028553009,
+      "rewards/margins": -0.06379567086696625,
+      "rewards/rejected": -0.33176031708717346,
+      "step": 56
+    },
+    {
+      "epoch": 0.6422535211267606,
+      "grad_norm": 8.948927879333496,
+      "learning_rate": 3.179487179487179e-06,
+      "log_odds_chosen": -0.0703490823507309,
+      "log_odds_ratio": -0.8050236105918884,
+      "logits/chosen": -2.2200632095336914,
+      "logits/rejected": -2.263523817062378,
+      "logps/chosen": -3.7273356914520264,
+      "logps/rejected": -3.654778480529785,
+      "loss": 4.0303,
+      "nll_loss": 3.9498226642608643,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.37273356318473816,
+      "rewards/margins": -0.00725569948554039,
+      "rewards/rejected": -0.36547785997390747,
+      "step": 57
+    },
+    {
+      "epoch": 0.6535211267605634,
+      "grad_norm": 9.45151138305664,
+      "learning_rate": 3.076923076923077e-06,
+      "log_odds_chosen": -0.11648232489824295,
+      "log_odds_ratio": -0.8264567852020264,
+      "logits/chosen": -2.264216184616089,
+      "logits/rejected": -2.270479202270508,
+      "logps/chosen": -3.7268123626708984,
+      "logps/rejected": -3.617461681365967,
+      "loss": 4.3405,
+      "nll_loss": 4.257853984832764,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.37268123030662537,
+      "rewards/margins": -0.010935075581073761,
+      "rewards/rejected": -0.3617461919784546,
+      "step": 58
+    },
+    {
+      "epoch": 0.6647887323943662,
+      "grad_norm": 13.517334938049316,
+      "learning_rate": 2.974358974358974e-06,
+      "log_odds_chosen": -0.04971995949745178,
+      "log_odds_ratio": -0.8288277387619019,
+      "logits/chosen": -2.0212604999542236,
+      "logits/rejected": -2.118274211883545,
+      "logps/chosen": -3.4767675399780273,
+      "logps/rejected": -3.431030035018921,
+      "loss": 3.3954,
+      "nll_loss": 3.312514066696167,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.34767675399780273,
+      "rewards/margins": -0.004573751240968704,
+      "rewards/rejected": -0.34310299158096313,
+      "step": 59
+    },
+    {
+      "epoch": 0.676056338028169,
+      "grad_norm": 10.096856117248535,
+      "learning_rate": 2.8717948717948717e-06,
+      "log_odds_chosen": 0.0826490968465805,
+      "log_odds_ratio": -0.8434144258499146,
+      "logits/chosen": -2.0799660682678223,
+      "logits/rejected": -2.1399242877960205,
+      "logps/chosen": -3.7268831729888916,
+      "logps/rejected": -3.8236966133117676,
+      "loss": 3.7636,
+      "nll_loss": 3.679297685623169,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.372688353061676,
+      "rewards/margins": 0.009681336581707,
+      "rewards/rejected": -0.38236966729164124,
+      "step": 60
+    },
+    {
+      "epoch": 0.6873239436619718,
+      "grad_norm": 11.962920188903809,
+      "learning_rate": 2.769230769230769e-06,
+      "log_odds_chosen": 0.11713685840368271,
+      "log_odds_ratio": -0.7304390072822571,
+      "logits/chosen": -2.0528836250305176,
+      "logits/rejected": -2.1740870475769043,
+      "logps/chosen": -3.7169508934020996,
+      "logps/rejected": -3.827744960784912,
+      "loss": 3.7201,
+      "nll_loss": 3.6470143795013428,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3716951012611389,
+      "rewards/margins": 0.011079397052526474,
+      "rewards/rejected": -0.3827744722366333,
+      "step": 61
+    },
+    {
+      "epoch": 0.6985915492957746,
+      "grad_norm": 9.109440803527832,
+      "learning_rate": 2.6666666666666664e-06,
+      "log_odds_chosen": -0.16476285457611084,
+      "log_odds_ratio": -0.902513325214386,
+      "logits/chosen": -2.2049403190612793,
+      "logits/rejected": -2.266833543777466,
+      "logps/chosen": -3.761657476425171,
+      "logps/rejected": -3.6014232635498047,
+      "loss": 3.9372,
+      "nll_loss": 3.846921443939209,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3761657476425171,
+      "rewards/margins": -0.016023436561226845,
+      "rewards/rejected": -0.3601423501968384,
+      "step": 62
+    },
+    {
+      "epoch": 0.7098591549295775,
+      "grad_norm": 10.382341384887695,
+      "learning_rate": 2.5641025641025644e-06,
+      "log_odds_chosen": -0.4335900843143463,
+      "log_odds_ratio": -1.0794651508331299,
+      "logits/chosen": -2.0837039947509766,
+      "logits/rejected": -2.1245696544647217,
+      "logps/chosen": -3.4378552436828613,
+      "logps/rejected": -3.0201077461242676,
+      "loss": 3.6002,
+      "nll_loss": 3.492260456085205,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.34378552436828613,
+      "rewards/margins": -0.04177476093173027,
+      "rewards/rejected": -0.30201077461242676,
+      "step": 63
+    },
+    {
+      "epoch": 0.7211267605633803,
+      "grad_norm": 12.012513160705566,
+      "learning_rate": 2.4615384615384615e-06,
+      "log_odds_chosen": -0.21123048663139343,
+      "log_odds_ratio": -0.8581590056419373,
+      "logits/chosen": -2.011561155319214,
+      "logits/rejected": -2.095082998275757,
+      "logps/chosen": -3.706972599029541,
+      "logps/rejected": -3.507680654525757,
+      "loss": 3.5491,
+      "nll_loss": 3.4632492065429688,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.3706972599029541,
+      "rewards/margins": -0.01992921158671379,
+      "rewards/rejected": -0.3507680296897888,
+      "step": 64
+    },
+    {
+      "epoch": 0.7323943661971831,
+      "grad_norm": 10.67837142944336,
+      "learning_rate": 2.358974358974359e-06,
+      "log_odds_chosen": 0.16844214498996735,
+      "log_odds_ratio": -0.6445605754852295,
+      "logits/chosen": -2.1053593158721924,
+      "logits/rejected": -2.14455509185791,
+      "logps/chosen": -3.4388720989227295,
+      "logps/rejected": -3.601630210876465,
+      "loss": 3.4626,
+      "nll_loss": 3.398159980773926,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.34388723969459534,
+      "rewards/margins": 0.016275810077786446,
+      "rewards/rejected": -0.36016303300857544,
+      "step": 65
+    },
+    {
+      "epoch": 0.7436619718309859,
+      "grad_norm": 14.22414493560791,
+      "learning_rate": 2.2564102564102562e-06,
+      "log_odds_chosen": -0.837117075920105,
+      "log_odds_ratio": -1.3559041023254395,
+      "logits/chosen": -2.200282573699951,
+      "logits/rejected": -2.2180466651916504,
+      "logps/chosen": -4.45404052734375,
+      "logps/rejected": -3.649200916290283,
+      "loss": 4.4569,
+      "nll_loss": 4.321296215057373,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4454040825366974,
+      "rewards/margins": -0.08048398792743683,
+      "rewards/rejected": -0.36492010951042175,
+      "step": 66
+    },
+    {
+      "epoch": 0.7549295774647887,
+      "grad_norm": 12.72874641418457,
+      "learning_rate": 2.1538461538461538e-06,
+      "log_odds_chosen": -0.9214501976966858,
+      "log_odds_ratio": -1.3725779056549072,
+      "logits/chosen": -2.1959099769592285,
+      "logits/rejected": -2.1905465126037598,
+      "logps/chosen": -4.213864803314209,
+      "logps/rejected": -3.309765577316284,
+      "loss": 4.0375,
+      "nll_loss": 3.900239944458008,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.4213864803314209,
+      "rewards/margins": -0.09040991961956024,
+      "rewards/rejected": -0.33097654581069946,
+      "step": 67
+    },
+    {
+      "epoch": 0.7661971830985915,
+      "grad_norm": 11.578482627868652,
+      "learning_rate": 2.051282051282051e-06,
+      "log_odds_chosen": -0.3243113160133362,
+      "log_odds_ratio": -0.9195367097854614,
+      "logits/chosen": -2.1235108375549316,
+      "logits/rejected": -2.168004035949707,
+      "logps/chosen": -3.7418808937072754,
+      "logps/rejected": -3.425140142440796,
+      "loss": 3.4577,
+      "nll_loss": 3.365786075592041,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.374188095331192,
+      "rewards/margins": -0.031674087047576904,
+      "rewards/rejected": -0.3425140380859375,
+      "step": 68
+    },
+    {
+      "epoch": 0.7774647887323943,
+      "grad_norm": 12.581503868103027,
+      "learning_rate": 1.9487179487179485e-06,
+      "log_odds_chosen": -0.03559496998786926,
+      "log_odds_ratio": -0.8679975271224976,
+      "logits/chosen": -1.9304691553115845,
+      "logits/rejected": -2.009279727935791,
+      "logps/chosen": -3.82916522026062,
+      "logps/rejected": -3.7897675037384033,
+      "loss": 3.6078,
+      "nll_loss": 3.520955801010132,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.38291651010513306,
+      "rewards/margins": -0.00393977016210556,
+      "rewards/rejected": -0.3789767622947693,
+      "step": 69
+    },
+    {
+      "epoch": 0.7887323943661971,
+      "grad_norm": 10.235701560974121,
+      "learning_rate": 1.8461538461538462e-06,
+      "log_odds_chosen": 0.7963358759880066,
+      "log_odds_ratio": -0.4532206356525421,
+      "logits/chosen": -2.097158908843994,
+      "logits/rejected": -2.1584880352020264,
+      "logps/chosen": -3.211224317550659,
+      "logps/rejected": -3.9823241233825684,
+      "loss": 3.4753,
+      "nll_loss": 3.4299583435058594,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.321122407913208,
+      "rewards/margins": 0.07711000740528107,
+      "rewards/rejected": -0.39823243021965027,
+      "step": 70
+    },
+    {
+      "epoch": 0.8,
+      "grad_norm": 10.014704704284668,
+      "learning_rate": 1.7435897435897436e-06,
+      "log_odds_chosen": -0.49982893466949463,
+      "log_odds_ratio": -1.097215175628662,
+      "logits/chosen": -2.100175380706787,
+      "logits/rejected": -2.1770012378692627,
+      "logps/chosen": -3.92124080657959,
+      "logps/rejected": -3.4407193660736084,
+      "loss": 3.9242,
+      "nll_loss": 3.8145065307617188,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.39212408661842346,
+      "rewards/margins": -0.04805214703083038,
+      "rewards/rejected": -0.3440719544887543,
+      "step": 71
+    },
+    {
+      "epoch": 0.8112676056338028,
+      "grad_norm": 11.128293991088867,
+      "learning_rate": 1.641025641025641e-06,
+      "log_odds_chosen": -0.03216097131371498,
+      "log_odds_ratio": -0.7252599000930786,
+      "logits/chosen": -1.9201287031173706,
+      "logits/rejected": -2.0173075199127197,
+      "logps/chosen": -3.436187744140625,
+      "logps/rejected": -3.4034676551818848,
+      "loss": 3.3486,
+      "nll_loss": 3.2760744094848633,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3436187505722046,
+      "rewards/margins": -0.0032719969749450684,
+      "rewards/rejected": -0.3403467535972595,
+      "step": 72
+    },
+    {
+      "epoch": 0.8112676056338028,
+      "eval_log_odds_chosen": 0.06351960450410843,
+      "eval_log_odds_ratio": -0.7638184428215027,
+      "eval_logits/chosen": -2.0969061851501465,
+      "eval_logits/rejected": -2.098085880279541,
+      "eval_logps/chosen": -3.975425958633423,
+      "eval_logps/rejected": -4.037297248840332,
+      "eval_loss": 4.211479187011719,
+      "eval_nll_loss": 4.135097503662109,
+      "eval_rewards/accuracies": 0.5,
+      "eval_rewards/chosen": -0.3975425958633423,
+      "eval_rewards/margins": 0.006187114864587784,
+      "eval_rewards/rejected": -0.40372970700263977,
+      "eval_runtime": 5.3655,
+      "eval_samples_per_second": 1.491,
+      "eval_steps_per_second": 0.746,
+      "step": 72
+    },
+    {
+      "epoch": 0.8225352112676056,
+      "grad_norm": 13.299338340759277,
+      "learning_rate": 1.5384615384615385e-06,
+      "log_odds_chosen": -0.08484344184398651,
+      "log_odds_ratio": -0.794756293296814,
+      "logits/chosen": -1.9263019561767578,
+      "logits/rejected": -1.9672499895095825,
+      "logps/chosen": -3.4308979511260986,
+      "logps/rejected": -3.347562789916992,
+      "loss": 3.225,
+      "nll_loss": 3.145517349243164,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3430897891521454,
+      "rewards/margins": -0.008333522826433182,
+      "rewards/rejected": -0.3347562849521637,
+      "step": 73
+    },
+    {
+      "epoch": 0.8338028169014085,
+      "grad_norm": 10.876466751098633,
+      "learning_rate": 1.4358974358974359e-06,
+      "log_odds_chosen": -0.16019976139068604,
+      "log_odds_ratio": -0.8210447430610657,
+      "logits/chosen": -1.9499791860580444,
+      "logits/rejected": -2.0576226711273193,
+      "logps/chosen": -3.4298863410949707,
+      "logps/rejected": -3.2793407440185547,
+      "loss": 3.2026,
+      "nll_loss": 3.1205201148986816,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.34298866987228394,
+      "rewards/margins": -0.015054592862725258,
+      "rewards/rejected": -0.32793405652046204,
+      "step": 74
+    },
+    {
+      "epoch": 0.8450704225352113,
+      "grad_norm": 11.573190689086914,
+      "learning_rate": 1.3333333333333332e-06,
+      "log_odds_chosen": -0.4442010223865509,
+      "log_odds_ratio": -0.9989137053489685,
+      "logits/chosen": -2.0589771270751953,
+      "logits/rejected": -2.151106595993042,
+      "logps/chosen": -3.764126777648926,
+      "logps/rejected": -3.340824604034424,
+      "loss": 3.6574,
+      "nll_loss": 3.557495594024658,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.37641265988349915,
+      "rewards/margins": -0.042330194264650345,
+      "rewards/rejected": -0.3340824842453003,
+      "step": 75
+    },
+    {
+      "epoch": 0.856338028169014,
+      "grad_norm": 10.903554916381836,
+      "learning_rate": 1.2307692307692308e-06,
+      "log_odds_chosen": -0.5089474320411682,
+      "log_odds_ratio": -1.0521657466888428,
+      "logits/chosen": -2.061066150665283,
+      "logits/rejected": -2.086852550506592,
+      "logps/chosen": -3.958143711090088,
+      "logps/rejected": -3.4646599292755127,
+      "loss": 3.9915,
+      "nll_loss": 3.886314868927002,
+      "rewards/accuracies": 0.125,
+      "rewards/chosen": -0.39581435918807983,
+      "rewards/margins": -0.049348361790180206,
+      "rewards/rejected": -0.3464660048484802,
+      "step": 76
+    },
+    {
+      "epoch": 0.8676056338028169,
+      "grad_norm": 13.457832336425781,
+      "learning_rate": 1.1282051282051281e-06,
+      "log_odds_chosen": 0.047474101185798645,
+      "log_odds_ratio": -0.7185592651367188,
+      "logits/chosen": -1.970052719116211,
+      "logits/rejected": -2.0962562561035156,
+      "logps/chosen": -3.3302972316741943,
+      "logps/rejected": -3.3730051517486572,
+      "loss": 3.4269,
+      "nll_loss": 3.3550257682800293,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.33302971720695496,
+      "rewards/margins": 0.004270805045962334,
+      "rewards/rejected": -0.33730053901672363,
+      "step": 77
+    },
+    {
+      "epoch": 0.8788732394366198,
+      "grad_norm": 13.770342826843262,
+      "learning_rate": 1.0256410256410255e-06,
+      "log_odds_chosen": 0.29137304425239563,
+      "log_odds_ratio": -0.7174768447875977,
+      "logits/chosen": -1.9147363901138306,
+      "logits/rejected": -2.074281692504883,
+      "logps/chosen": -3.6085116863250732,
+      "logps/rejected": -3.9022023677825928,
+      "loss": 3.6132,
+      "nll_loss": 3.541477680206299,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.3608511686325073,
+      "rewards/margins": 0.02936907671391964,
+      "rewards/rejected": -0.3902202248573303,
+      "step": 78
+    },
+    {
+      "epoch": 0.8901408450704226,
+      "grad_norm": 11.069844245910645,
+      "learning_rate": 9.230769230769231e-07,
+      "log_odds_chosen": -0.5195303559303284,
+      "log_odds_ratio": -1.1289513111114502,
+      "logits/chosen": -2.013373851776123,
+      "logits/rejected": -2.119136333465576,
+      "logps/chosen": -4.341013431549072,
+      "logps/rejected": -3.829310178756714,
+      "loss": 4.1473,
+      "nll_loss": 4.034441947937012,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.43410131335258484,
+      "rewards/margins": -0.05117032676935196,
+      "rewards/rejected": -0.3829309940338135,
+      "step": 79
+    },
+    {
+      "epoch": 0.9014084507042254,
+      "grad_norm": 11.87746810913086,
+      "learning_rate": 8.205128205128205e-07,
+      "log_odds_chosen": -0.13725686073303223,
+      "log_odds_ratio": -0.8136627674102783,
+      "logits/chosen": -1.9058239459991455,
+      "logits/rejected": -2.0132644176483154,
+      "logps/chosen": -3.670649528503418,
+      "logps/rejected": -3.545762062072754,
+      "loss": 3.5501,
+      "nll_loss": 3.46868634223938,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.3670649826526642,
+      "rewards/margins": -0.01248876191675663,
+      "rewards/rejected": -0.3545762300491333,
+      "step": 80
+    },
+    {
+      "epoch": 0.9126760563380282,
+      "grad_norm": 14.643924713134766,
+      "learning_rate": 7.179487179487179e-07,
+      "log_odds_chosen": 0.27706030011177063,
+      "log_odds_ratio": -0.6745563745498657,
+      "logits/chosen": -1.9193668365478516,
+      "logits/rejected": -2.0037789344787598,
+      "logps/chosen": -3.511770725250244,
+      "logps/rejected": -3.7823686599731445,
+      "loss": 3.4836,
+      "nll_loss": 3.416135311126709,
+      "rewards/accuracies": 0.75,
+      "rewards/chosen": -0.3511770963668823,
+      "rewards/margins": 0.027059808373451233,
+      "rewards/rejected": -0.37823688983917236,
+      "step": 81
+    },
+    {
+      "epoch": 0.923943661971831,
+      "grad_norm": 11.816194534301758,
+      "learning_rate": 6.153846153846154e-07,
+      "log_odds_chosen": -0.5967518091201782,
+      "log_odds_ratio": -1.1597626209259033,
+      "logits/chosen": -2.0161638259887695,
+      "logits/rejected": -2.067549228668213,
+      "logps/chosen": -3.8716623783111572,
+      "logps/rejected": -3.2700958251953125,
+      "loss": 4.0015,
+      "nll_loss": 3.8855390548706055,
+      "rewards/accuracies": 0.25,
+      "rewards/chosen": -0.38716626167297363,
+      "rewards/margins": -0.06015666574239731,
+      "rewards/rejected": -0.3270095884799957,
+      "step": 82
+    },
+    {
+      "epoch": 0.9352112676056338,
+      "grad_norm": 11.173247337341309,
+      "learning_rate": 5.128205128205127e-07,
+      "log_odds_chosen": -0.45643556118011475,
+      "log_odds_ratio": -1.0668988227844238,
+      "logits/chosen": -2.018862009048462,
+      "logits/rejected": -2.0817437171936035,
+      "logps/chosen": -4.08735466003418,
+      "logps/rejected": -3.6331050395965576,
+      "loss": 3.8471,
+      "nll_loss": 3.740403890609741,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.4087355136871338,
+      "rewards/margins": -0.04542497545480728,
+      "rewards/rejected": -0.3633105158805847,
+      "step": 83
+    },
+    {
+      "epoch": 0.9464788732394366,
+      "grad_norm": 12.033004760742188,
+      "learning_rate": 4.1025641025641024e-07,
+      "log_odds_chosen": -0.16924908757209778,
+      "log_odds_ratio": -0.8661594390869141,
+      "logits/chosen": -1.8825433254241943,
+      "logits/rejected": -1.9633598327636719,
+      "logps/chosen": -3.092277765274048,
+      "logps/rejected": -2.939509868621826,
+      "loss": 2.9276,
+      "nll_loss": 2.840937614440918,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.3092277944087982,
+      "rewards/margins": -0.015276828780770302,
+      "rewards/rejected": -0.29395097494125366,
+      "step": 84
+    },
+    {
+      "epoch": 0.9577464788732394,
+      "grad_norm": 11.308173179626465,
+      "learning_rate": 3.076923076923077e-07,
+      "log_odds_chosen": -0.5136163830757141,
+      "log_odds_ratio": -1.1063024997711182,
+      "logits/chosen": -1.9425265789031982,
+      "logits/rejected": -2.0414481163024902,
+      "logps/chosen": -3.515744924545288,
+      "logps/rejected": -3.021228790283203,
+      "loss": 3.4749,
+      "nll_loss": 3.364316940307617,
+      "rewards/accuracies": 0.5,
+      "rewards/chosen": -0.35157451033592224,
+      "rewards/margins": -0.04945161193609238,
+      "rewards/rejected": -0.30212289094924927,
+      "step": 85
+    },
+    {
+      "epoch": 0.9690140845070423,
+      "grad_norm": 11.616045951843262,
+      "learning_rate": 2.0512820512820512e-07,
+      "log_odds_chosen": -0.21424178779125214,
+      "log_odds_ratio": -0.8350873589515686,
+      "logits/chosen": -1.8886396884918213,
+      "logits/rejected": -1.9696400165557861,
+      "logps/chosen": -3.475510358810425,
+      "logps/rejected": -3.2696402072906494,
+      "loss": 3.5142,
+      "nll_loss": 3.430680990219116,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.34755104780197144,
+      "rewards/margins": -0.020586999133229256,
+      "rewards/rejected": -0.32696402072906494,
+      "step": 86
+    },
+    {
+      "epoch": 0.9802816901408451,
+      "grad_norm": 11.651595115661621,
+      "learning_rate": 1.0256410256410256e-07,
+      "log_odds_chosen": 0.28179410099983215,
+      "log_odds_ratio": -0.655161440372467,
+      "logits/chosen": -1.94129478931427,
+      "logits/rejected": -1.9500186443328857,
+      "logps/chosen": -2.9875540733337402,
+      "logps/rejected": -3.257594108581543,
+      "loss": 2.9425,
+      "nll_loss": 2.8769569396972656,
+      "rewards/accuracies": 0.625,
+      "rewards/chosen": -0.298755407333374,
+      "rewards/margins": 0.027004040777683258,
+      "rewards/rejected": -0.3257594108581543,
+      "step": 87
+    },
+    {
+      "epoch": 0.9915492957746479,
+      "grad_norm": 11.723467826843262,
+      "learning_rate": 0.0,
+      "log_odds_chosen": -0.272980272769928,
+      "log_odds_ratio": -0.9368327856063843,
+      "logits/chosen": -1.8292593955993652,
+      "logits/rejected": -1.9691557884216309,
+      "logps/chosen": -3.174346923828125,
+      "logps/rejected": -2.9293620586395264,
+      "loss": 2.9493,
+      "nll_loss": 2.855593204498291,
+      "rewards/accuracies": 0.375,
+      "rewards/chosen": -0.3174346685409546,
+      "rewards/margins": -0.024498462677001953,
+      "rewards/rejected": -0.292936235666275,
+      "step": 88
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 88,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 1,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 0.0,
+  "train_batch_size": 2,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7427e2a16bd6d7bbb4c3410e92f2a24d73d41d3769a24765bdd6d9b08a6abeae
+size 5368