VPO-Pess-Zephyr-7B-0.0001-iter-3 / trainer_state.json
Katayoon's picture
Model save
e9403b8 verified
{
"best_metric": null,
"best_model_checkpoint": null,
"epoch": 0.9874476987447699,
"eval_steps": 500,
"global_step": 59,
"is_hyper_param_search": false,
"is_local_process_zero": true,
"is_world_process_zero": true,
"log_history": [
{
"epoch": 0.016736401673640166,
"grad_norm": 6.471892187679766,
"learning_rate": 1.6666666666666664e-08,
"logits/chosen": -2.7078990936279297,
"logits/rejected": -2.7151172161102295,
"logps/chosen": -156.76602172851562,
"logps/pi_response": -170.17237854003906,
"logps/ref_response": -170.17237854003906,
"logps/rejected": -165.455078125,
"loss": 0.6933,
"rewards/accuracies": 0.0,
"rewards/chosen": 0.0,
"rewards/margins": 0.0,
"rewards/rejected": 0.0,
"step": 1
},
{
"epoch": 0.16736401673640167,
"grad_norm": 6.194703133950661,
"learning_rate": 9.860114570402053e-08,
"logits/chosen": -2.658425807952881,
"logits/rejected": -2.664215564727783,
"logps/chosen": -134.5729522705078,
"logps/pi_response": -136.7792205810547,
"logps/ref_response": -136.80030822753906,
"logps/rejected": -139.36070251464844,
"loss": 0.6933,
"rewards/accuracies": 0.4236111044883728,
"rewards/chosen": 5.2155293815303594e-05,
"rewards/margins": 0.00010491909779375419,
"rewards/rejected": -5.276386218611151e-05,
"step": 10
},
{
"epoch": 0.33472803347280333,
"grad_norm": 6.481693690890403,
"learning_rate": 8.374915007591053e-08,
"logits/chosen": -2.587273120880127,
"logits/rejected": -2.5810611248016357,
"logps/chosen": -144.60513305664062,
"logps/pi_response": -143.98855590820312,
"logps/ref_response": -143.8643341064453,
"logps/rejected": -143.2288055419922,
"loss": 0.6932,
"rewards/accuracies": 0.534375011920929,
"rewards/chosen": -0.0007258538389578462,
"rewards/margins": 0.000704266014508903,
"rewards/rejected": -0.0014301198534667492,
"step": 20
},
{
"epoch": 0.502092050209205,
"grad_norm": 6.9528073416896525,
"learning_rate": 5.738232820012406e-08,
"logits/chosen": -2.5854544639587402,
"logits/rejected": -2.580871105194092,
"logps/chosen": -134.29434204101562,
"logps/pi_response": -136.11817932128906,
"logps/ref_response": -135.50979614257812,
"logps/rejected": -138.55838012695312,
"loss": 0.6929,
"rewards/accuracies": 0.515625,
"rewards/chosen": -0.006442646030336618,
"rewards/margins": 2.6706849894253537e-05,
"rewards/rejected": -0.006469352636486292,
"step": 30
},
{
"epoch": 0.6694560669456067,
"grad_norm": 6.268077183061405,
"learning_rate": 2.8496739886173992e-08,
"logits/chosen": -2.6380069255828857,
"logits/rejected": -2.631618022918701,
"logps/chosen": -152.0018768310547,
"logps/pi_response": -151.4007568359375,
"logps/ref_response": -150.2733917236328,
"logps/rejected": -149.3154754638672,
"loss": 0.6933,
"rewards/accuracies": 0.4781250059604645,
"rewards/chosen": -0.01104762963950634,
"rewards/margins": -0.00010848313104361296,
"rewards/rejected": -0.01093914732336998,
"step": 40
},
{
"epoch": 0.8368200836820083,
"grad_norm": 6.590783839049672,
"learning_rate": 6.947819411632222e-09,
"logits/chosen": -2.62695574760437,
"logits/rejected": -2.6248316764831543,
"logps/chosen": -145.61465454101562,
"logps/pi_response": -151.46591186523438,
"logps/ref_response": -150.24114990234375,
"logps/rejected": -151.93894958496094,
"loss": 0.6928,
"rewards/accuracies": 0.5718749761581421,
"rewards/chosen": -0.01102914847433567,
"rewards/margins": 0.0005052736960351467,
"rewards/rejected": -0.011534422636032104,
"step": 50
},
{
"epoch": 0.9874476987447699,
"step": 59,
"total_flos": 0.0,
"train_loss": 0.6930160360821223,
"train_runtime": 2638.5215,
"train_samples_per_second": 5.792,
"train_steps_per_second": 0.022
}
],
"logging_steps": 10,
"max_steps": 59,
"num_input_tokens_seen": 0,
"num_train_epochs": 1,
"save_steps": 100,
"total_flos": 0.0,
"train_batch_size": 4,
"trial_name": null,
"trial_params": null
}