| { | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.9984301412872841, | |
| "eval_steps": 500, | |
| "global_step": 159, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "epoch": 0.006279434850863423, | |
| "grad_norm": 6.26599976561471, | |
| "learning_rate": 3.125e-08, | |
| "logits/chosen": 2.506397247314453, | |
| "logits/rejected": 2.9004628658294678, | |
| "logps/chosen": -304.1236877441406, | |
| "logps/pi_response": -206.54444885253906, | |
| "logps/ref_response": -206.54444885253906, | |
| "logps/rejected": -435.8030700683594, | |
| "loss": 0.6931, | |
| "rewards/accuracies": 0.0, | |
| "rewards/chosen": 0.0, | |
| "rewards/margins": 0.0, | |
| "rewards/rejected": 0.0, | |
| "step": 1 | |
| }, | |
| { | |
| "epoch": 0.06279434850863422, | |
| "grad_norm": 6.5208769318369715, | |
| "learning_rate": 3.1249999999999997e-07, | |
| "logits/chosen": 2.331634759902954, | |
| "logits/rejected": 2.5180442333221436, | |
| "logps/chosen": -271.9783935546875, | |
| "logps/pi_response": -178.7827606201172, | |
| "logps/ref_response": -178.90916442871094, | |
| "logps/rejected": -350.5011901855469, | |
| "loss": 0.6933, | |
| "rewards/accuracies": 0.4305555522441864, | |
| "rewards/chosen": 0.0012251290027052164, | |
| "rewards/margins": 0.0004269891942385584, | |
| "rewards/rejected": 0.0007981394301168621, | |
| "step": 10 | |
| }, | |
| { | |
| "epoch": 0.12558869701726844, | |
| "grad_norm": 6.149140437491673, | |
| "learning_rate": 4.990353313429303e-07, | |
| "logits/chosen": 2.4192910194396973, | |
| "logits/rejected": 2.5796756744384766, | |
| "logps/chosen": -264.28521728515625, | |
| "logps/pi_response": -169.31643676757812, | |
| "logps/ref_response": -172.20350646972656, | |
| "logps/rejected": -353.85626220703125, | |
| "loss": 0.6857, | |
| "rewards/accuracies": 0.643750011920929, | |
| "rewards/chosen": 0.020391147583723068, | |
| "rewards/margins": 0.01745672896504402, | |
| "rewards/rejected": 0.002934417687356472, | |
| "step": 20 | |
| }, | |
| { | |
| "epoch": 0.18838304552590268, | |
| "grad_norm": 5.838815655099781, | |
| "learning_rate": 4.882681251368548e-07, | |
| "logits/chosen": 2.2606046199798584, | |
| "logits/rejected": 2.399362087249756, | |
| "logps/chosen": -261.7556457519531, | |
| "logps/pi_response": -151.37615966796875, | |
| "logps/ref_response": -163.4231719970703, | |
| "logps/rejected": -365.0315246582031, | |
| "loss": 0.6617, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": 0.08448224514722824, | |
| "rewards/margins": 0.08167780935764313, | |
| "rewards/rejected": 0.002804456278681755, | |
| "step": 30 | |
| }, | |
| { | |
| "epoch": 0.25117739403453687, | |
| "grad_norm": 6.251630948090285, | |
| "learning_rate": 4.6604720940421207e-07, | |
| "logits/chosen": 2.2731361389160156, | |
| "logits/rejected": 2.4390079975128174, | |
| "logps/chosen": -283.78436279296875, | |
| "logps/pi_response": -162.61363220214844, | |
| "logps/ref_response": -179.4296417236328, | |
| "logps/rejected": -353.84832763671875, | |
| "loss": 0.6346, | |
| "rewards/accuracies": 0.65625, | |
| "rewards/chosen": 0.11685456335544586, | |
| "rewards/margins": 0.13428333401679993, | |
| "rewards/rejected": -0.01742876134812832, | |
| "step": 40 | |
| }, | |
| { | |
| "epoch": 0.3139717425431711, | |
| "grad_norm": 6.204506278337755, | |
| "learning_rate": 4.3344075855595097e-07, | |
| "logits/chosen": 1.9736655950546265, | |
| "logits/rejected": 2.127274513244629, | |
| "logps/chosen": -262.3057861328125, | |
| "logps/pi_response": -151.85891723632812, | |
| "logps/ref_response": -168.13865661621094, | |
| "logps/rejected": -346.0932922363281, | |
| "loss": 0.6134, | |
| "rewards/accuracies": 0.731249988079071, | |
| "rewards/chosen": 0.028540844097733498, | |
| "rewards/margins": 0.18684375286102295, | |
| "rewards/rejected": -0.1583028882741928, | |
| "step": 50 | |
| }, | |
| { | |
| "epoch": 0.37676609105180536, | |
| "grad_norm": 5.062174246506226, | |
| "learning_rate": 3.920161866827889e-07, | |
| "logits/chosen": 1.7557458877563477, | |
| "logits/rejected": 1.9249584674835205, | |
| "logps/chosen": -270.45989990234375, | |
| "logps/pi_response": -142.9950714111328, | |
| "logps/ref_response": -163.70513916015625, | |
| "logps/rejected": -408.8035583496094, | |
| "loss": 0.5803, | |
| "rewards/accuracies": 0.7562500238418579, | |
| "rewards/chosen": -0.011763873510062695, | |
| "rewards/margins": 0.3801829516887665, | |
| "rewards/rejected": -0.39194679260253906, | |
| "step": 60 | |
| }, | |
| { | |
| "epoch": 0.43956043956043955, | |
| "grad_norm": 4.514633956267623, | |
| "learning_rate": 3.4376480090239047e-07, | |
| "logits/chosen": 1.6569864749908447, | |
| "logits/rejected": 1.8295177221298218, | |
| "logps/chosen": -254.6250457763672, | |
| "logps/pi_response": -151.6630859375, | |
| "logps/ref_response": -169.66830444335938, | |
| "logps/rejected": -397.61846923828125, | |
| "loss": 0.5536, | |
| "rewards/accuracies": 0.6875, | |
| "rewards/chosen": -0.03449697047472, | |
| "rewards/margins": 0.402763307094574, | |
| "rewards/rejected": -0.437260240316391, | |
| "step": 70 | |
| }, | |
| { | |
| "epoch": 0.5023547880690737, | |
| "grad_norm": 5.51673658153614, | |
| "learning_rate": 2.910060778827554e-07, | |
| "logits/chosen": 1.7929670810699463, | |
| "logits/rejected": 1.9936065673828125, | |
| "logps/chosen": -301.0906677246094, | |
| "logps/pi_response": -161.8425750732422, | |
| "logps/ref_response": -179.93528747558594, | |
| "logps/rejected": -400.73992919921875, | |
| "loss": 0.5737, | |
| "rewards/accuracies": 0.737500011920929, | |
| "rewards/chosen": -0.09138133376836777, | |
| "rewards/margins": 0.43019238114356995, | |
| "rewards/rejected": -0.5215736627578735, | |
| "step": 80 | |
| }, | |
| { | |
| "epoch": 0.565149136577708, | |
| "grad_norm": 4.60951328858753, | |
| "learning_rate": 2.3627616503391812e-07, | |
| "logits/chosen": 1.629387617111206, | |
| "logits/rejected": 1.7540788650512695, | |
| "logps/chosen": -322.956787109375, | |
| "logps/pi_response": -162.01455688476562, | |
| "logps/ref_response": -183.2974853515625, | |
| "logps/rejected": -443.8954162597656, | |
| "loss": 0.5516, | |
| "rewards/accuracies": 0.699999988079071, | |
| "rewards/chosen": -0.10067585855722427, | |
| "rewards/margins": 0.5000637769699097, | |
| "rewards/rejected": -0.6007396578788757, | |
| "step": 90 | |
| }, | |
| { | |
| "epoch": 0.6279434850863422, | |
| "grad_norm": 5.912396924653729, | |
| "learning_rate": 1.8220596619089573e-07, | |
| "logits/chosen": 1.504276990890503, | |
| "logits/rejected": 1.6940498352050781, | |
| "logps/chosen": -276.9794921875, | |
| "logps/pi_response": -147.35867309570312, | |
| "logps/ref_response": -169.77037048339844, | |
| "logps/rejected": -378.74542236328125, | |
| "loss": 0.5559, | |
| "rewards/accuracies": 0.675000011920929, | |
| "rewards/chosen": -0.08432722836732864, | |
| "rewards/margins": 0.4150608479976654, | |
| "rewards/rejected": -0.4993881285190582, | |
| "step": 100 | |
| }, | |
| { | |
| "epoch": 0.6907378335949764, | |
| "grad_norm": 6.153587429768731, | |
| "learning_rate": 1.3139467229135998e-07, | |
| "logits/chosen": 1.5478519201278687, | |
| "logits/rejected": 1.7215379476547241, | |
| "logps/chosen": -282.90142822265625, | |
| "logps/pi_response": -164.94143676757812, | |
| "logps/ref_response": -186.05224609375, | |
| "logps/rejected": -429.9634704589844, | |
| "loss": 0.5423, | |
| "rewards/accuracies": 0.706250011920929, | |
| "rewards/chosen": -0.03883028030395508, | |
| "rewards/margins": 0.512156069278717, | |
| "rewards/rejected": -0.5509864091873169, | |
| "step": 110 | |
| }, | |
| { | |
| "epoch": 0.7535321821036107, | |
| "grad_norm": 5.201928644747849, | |
| "learning_rate": 8.628481651367875e-08, | |
| "logits/chosen": 1.4170118570327759, | |
| "logits/rejected": 1.6051431894302368, | |
| "logps/chosen": -289.0984802246094, | |
| "logps/pi_response": -153.79116821289062, | |
| "logps/ref_response": -173.28074645996094, | |
| "logps/rejected": -421.106689453125, | |
| "loss": 0.5484, | |
| "rewards/accuracies": 0.7749999761581421, | |
| "rewards/chosen": -0.0854920819401741, | |
| "rewards/margins": 0.5487265586853027, | |
| "rewards/rejected": -0.6342186331748962, | |
| "step": 120 | |
| }, | |
| { | |
| "epoch": 0.8163265306122449, | |
| "grad_norm": 5.723480822818505, | |
| "learning_rate": 4.904486005914027e-08, | |
| "logits/chosen": 1.4594230651855469, | |
| "logits/rejected": 1.6405918598175049, | |
| "logps/chosen": -275.15325927734375, | |
| "logps/pi_response": -150.0062713623047, | |
| "logps/ref_response": -171.3158721923828, | |
| "logps/rejected": -406.230224609375, | |
| "loss": 0.5302, | |
| "rewards/accuracies": 0.7437499761581421, | |
| "rewards/chosen": -0.06263308972120285, | |
| "rewards/margins": 0.5518540740013123, | |
| "rewards/rejected": -0.6144872307777405, | |
| "step": 130 | |
| }, | |
| { | |
| "epoch": 0.8791208791208791, | |
| "grad_norm": 6.184911001401629, | |
| "learning_rate": 2.1464952759020856e-08, | |
| "logits/chosen": 1.5727994441986084, | |
| "logits/rejected": 1.7528778314590454, | |
| "logps/chosen": -298.0870056152344, | |
| "logps/pi_response": -154.67698669433594, | |
| "logps/ref_response": -175.67694091796875, | |
| "logps/rejected": -445.009521484375, | |
| "loss": 0.5339, | |
| "rewards/accuracies": 0.793749988079071, | |
| "rewards/chosen": -0.09744294732809067, | |
| "rewards/margins": 0.645055890083313, | |
| "rewards/rejected": -0.7424989342689514, | |
| "step": 140 | |
| }, | |
| { | |
| "epoch": 0.9419152276295133, | |
| "grad_norm": 4.863112258557619, | |
| "learning_rate": 4.8708793644441086e-09, | |
| "logits/chosen": 1.399854302406311, | |
| "logits/rejected": 1.5590882301330566, | |
| "logps/chosen": -294.4033203125, | |
| "logps/pi_response": -164.02032470703125, | |
| "logps/ref_response": -185.34927368164062, | |
| "logps/rejected": -422.95574951171875, | |
| "loss": 0.5319, | |
| "rewards/accuracies": 0.800000011920929, | |
| "rewards/chosen": -0.10503934323787689, | |
| "rewards/margins": 0.6324479579925537, | |
| "rewards/rejected": -0.7374873757362366, | |
| "step": 150 | |
| }, | |
| { | |
| "epoch": 0.9984301412872841, | |
| "step": 159, | |
| "total_flos": 0.0, | |
| "train_loss": 0.5863969910819575, | |
| "train_runtime": 3000.191, | |
| "train_samples_per_second": 6.792, | |
| "train_steps_per_second": 0.053 | |
| } | |
| ], | |
| "logging_steps": 10, | |
| "max_steps": 159, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 100, | |
| "total_flos": 0.0, | |
| "train_batch_size": 4, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |