{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.0,
  "eval_steps": 100,
  "global_step": 32,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "debug/policy_chosen_logits": 1.5414493083953857,
      "debug/policy_chosen_logps": -465.5361328125,
      "debug/policy_rejected_logits": 1.6527317762374878,
      "debug/policy_rejected_logps": -427.4836120605469,
      "debug/reference_chosen_logps": -465.5361328125,
      "debug/reference_rejected_logps": -427.4836120605469,
      "epoch": 0.03125,
      "grad_norm": 12.323765072380743,
      "learning_rate": 1e-06,
      "logits/chosen": 1.5414493083953857,
      "logits/rejected": 1.6527317762374878,
      "logps/chosen": -465.5361328125,
      "logps/rejected": -427.4836120605469,
      "loss": 0.5,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1
    },
    {
      "debug/policy_chosen_logits": 1.8114635944366455,
      "debug/policy_chosen_logps": -395.85809326171875,
      "debug/policy_rejected_logits": 2.0628278255462646,
      "debug/policy_rejected_logps": -405.449462890625,
      "debug/reference_chosen_logps": -395.2831726074219,
      "debug/reference_rejected_logps": -405.4757080078125,
      "epoch": 0.0625,
      "grad_norm": 5.688989109980878,
      "learning_rate": 1e-06,
      "logits/chosen": 1.8114635944366455,
      "logits/rejected": 2.0628278255462646,
      "logps/chosen": -395.85809326171875,
      "logps/rejected": -405.449462890625,
      "loss": 0.5021,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.005749321077018976,
      "rewards/margins": -0.006011696066707373,
      "rewards/rejected": 0.0002623748732730746,
      "step": 2
    },
    {
      "debug/policy_chosen_logits": 1.734413504600525,
      "debug/policy_chosen_logps": -416.40167236328125,
      "debug/policy_rejected_logits": 1.5799931287765503,
      "debug/policy_rejected_logps": -398.26263427734375,
      "debug/reference_chosen_logps": -416.07275390625,
      "debug/reference_rejected_logps": -398.1192321777344,
      "epoch": 0.09375,
      "grad_norm": 3.905789416952891,
      "learning_rate": 1e-06,
      "logits/chosen": 1.734413504600525,
      "logits/rejected": 1.5799931287765503,
      "logps/chosen": -416.40167236328125,
      "logps/rejected": -398.26263427734375,
      "loss": 0.5007,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00328903179615736,
      "rewards/margins": -0.0018549349624663591,
      "rewards/rejected": -0.0014340970665216446,
      "step": 3
    },
    {
      "debug/policy_chosen_logits": 1.7133558988571167,
      "debug/policy_chosen_logps": -429.8788757324219,
      "debug/policy_rejected_logits": 1.561805009841919,
      "debug/policy_rejected_logps": -398.11846923828125,
      "debug/reference_chosen_logps": -429.8458557128906,
      "debug/reference_rejected_logps": -398.2086486816406,
      "epoch": 0.125,
      "grad_norm": 4.592860856362045,
      "learning_rate": 1e-06,
      "logits/chosen": 1.7133558988571167,
      "logits/rejected": 1.561805009841919,
      "logps/chosen": -429.8788757324219,
      "logps/rejected": -398.11846923828125,
      "loss": 0.5006,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.0003304291167296469,
      "rewards/margins": -0.0012322617694735527,
      "rewards/rejected": 0.0009018326527439058,
      "step": 4
    },
    {
      "debug/policy_chosen_logits": 1.6188408136367798,
      "debug/policy_chosen_logps": -405.05450439453125,
      "debug/policy_rejected_logits": 1.7664055824279785,
      "debug/policy_rejected_logps": -389.2750244140625,
      "debug/reference_chosen_logps": -404.82635498046875,
      "debug/reference_rejected_logps": -389.15972900390625,
      "epoch": 0.15625,
      "grad_norm": 4.105078966015891,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6188408136367798,
      "logits/rejected": 1.7664055824279785,
      "logps/chosen": -405.05450439453125,
      "logps/rejected": -389.2750244140625,
      "loss": 0.4992,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002281513065099716,
      "rewards/margins": -0.001129131531342864,
      "rewards/rejected": -0.0011523817665874958,
      "step": 5
    },
    {
      "debug/policy_chosen_logits": 1.5052284002304077,
      "debug/policy_chosen_logps": -417.0735778808594,
      "debug/policy_rejected_logits": 1.241634488105774,
      "debug/policy_rejected_logps": -425.4691467285156,
      "debug/reference_chosen_logps": -417.2569580078125,
      "debug/reference_rejected_logps": -425.63446044921875,
      "epoch": 0.1875,
      "grad_norm": 3.9533420030932884,
      "learning_rate": 1e-06,
      "logits/chosen": 1.5052284002304077,
      "logits/rejected": 1.241634488105774,
      "logps/chosen": -417.0735778808594,
      "logps/rejected": -425.4691467285156,
      "loss": 0.5005,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0018336105858907104,
      "rewards/margins": 0.0001804353087209165,
      "rewards/rejected": 0.0016531753353774548,
      "step": 6
    },
    {
      "debug/policy_chosen_logits": 1.5048482418060303,
      "debug/policy_chosen_logps": -410.84844970703125,
      "debug/policy_rejected_logits": 1.9221699237823486,
      "debug/policy_rejected_logps": -365.1915283203125,
      "debug/reference_chosen_logps": -411.6490478515625,
      "debug/reference_rejected_logps": -365.34991455078125,
      "epoch": 0.21875,
      "grad_norm": 4.1754933609078195,
      "learning_rate": 1e-06,
      "logits/chosen": 1.5048482418060303,
      "logits/rejected": 1.9221699237823486,
      "logps/chosen": -410.84844970703125,
      "logps/rejected": -365.1915283203125,
      "loss": 0.4994,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.008005637675523758,
      "rewards/margins": 0.006421546917408705,
      "rewards/rejected": 0.0015840912237763405,
      "step": 7
    },
    {
      "debug/policy_chosen_logits": 1.3764795064926147,
      "debug/policy_chosen_logps": -410.59136962890625,
      "debug/policy_rejected_logits": 1.3361552953720093,
      "debug/policy_rejected_logps": -446.6079406738281,
      "debug/reference_chosen_logps": -411.0892028808594,
      "debug/reference_rejected_logps": -446.5020446777344,
      "epoch": 0.25,
      "grad_norm": 28.29512163500699,
      "learning_rate": 1e-06,
      "logits/chosen": 1.3764795064926147,
      "logits/rejected": 1.3361552953720093,
      "logps/chosen": -410.59136962890625,
      "logps/rejected": -446.6079406738281,
      "loss": 0.4985,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.004978599026799202,
      "rewards/margins": 0.006037559360265732,
      "rewards/rejected": -0.0010589603334665298,
      "step": 8
    },
    {
      "debug/policy_chosen_logits": 1.610563039779663,
      "debug/policy_chosen_logps": -406.68865966796875,
      "debug/policy_rejected_logits": 1.793952226638794,
      "debug/policy_rejected_logps": -407.90484619140625,
      "debug/reference_chosen_logps": -406.7731018066406,
      "debug/reference_rejected_logps": -408.06695556640625,
      "epoch": 0.28125,
      "grad_norm": 8.342311302611428,
      "learning_rate": 1e-06,
      "logits/chosen": 1.610563039779663,
      "logits/rejected": 1.793952226638794,
      "logps/chosen": -406.68865966796875,
      "logps/rejected": -407.90484619140625,
      "loss": 0.4981,
      "rewards/accuracies": 0.375,
      "rewards/chosen": 0.0008446313440799713,
      "rewards/margins": -0.0007765005575492978,
      "rewards/rejected": 0.0016211319016292691,
      "step": 9
    },
    {
      "debug/policy_chosen_logits": 1.8192218542099,
      "debug/policy_chosen_logps": -399.52435302734375,
      "debug/policy_rejected_logits": 1.5653293132781982,
      "debug/policy_rejected_logps": -385.7630615234375,
      "debug/reference_chosen_logps": -399.35186767578125,
      "debug/reference_rejected_logps": -385.9487609863281,
      "epoch": 0.3125,
      "grad_norm": 4.2502799962912725,
      "learning_rate": 1e-06,
      "logits/chosen": 1.8192218542099,
      "logits/rejected": 1.5653293132781982,
      "logps/chosen": -399.52435302734375,
      "logps/rejected": -385.7630615234375,
      "loss": 0.5017,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.0017250444507226348,
      "rewards/margins": -0.003581695957109332,
      "rewards/rejected": 0.0018566511571407318,
      "step": 10
    },
    {
      "debug/policy_chosen_logits": 1.8145008087158203,
      "debug/policy_chosen_logps": -408.01934814453125,
      "debug/policy_rejected_logits": 1.7603477239608765,
      "debug/policy_rejected_logps": -435.0226135253906,
      "debug/reference_chosen_logps": -408.4494323730469,
      "debug/reference_rejected_logps": -435.68634033203125,
      "epoch": 0.34375,
      "grad_norm": 4.1787448496513075,
      "learning_rate": 1e-06,
      "logits/chosen": 1.8145008087158203,
      "logits/rejected": 1.7603477239608765,
      "logps/chosen": -408.01934814453125,
      "logps/rejected": -435.0226135253906,
      "loss": 0.4975,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.004300727508962154,
      "rewards/margins": -0.0023362734355032444,
      "rewards/rejected": 0.006637001410126686,
      "step": 11
    },
    {
      "debug/policy_chosen_logits": 1.6767597198486328,
      "debug/policy_chosen_logps": -384.9023742675781,
      "debug/policy_rejected_logits": 1.4636709690093994,
      "debug/policy_rejected_logps": -410.29254150390625,
      "debug/reference_chosen_logps": -384.634033203125,
      "debug/reference_rejected_logps": -410.1116943359375,
      "epoch": 0.375,
      "grad_norm": 4.319336871459386,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6767597198486328,
      "logits/rejected": 1.4636709690093994,
      "logps/chosen": -384.9023742675781,
      "logps/rejected": -410.29254150390625,
      "loss": 0.497,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.0026834101881831884,
      "rewards/margins": -0.0008746716193854809,
      "rewards/rejected": -0.0018087388016283512,
      "step": 12
    },
    {
      "debug/policy_chosen_logits": 1.508570909500122,
      "debug/policy_chosen_logps": -404.47314453125,
      "debug/policy_rejected_logits": 1.6251033544540405,
      "debug/policy_rejected_logps": -459.6101379394531,
      "debug/reference_chosen_logps": -404.60076904296875,
      "debug/reference_rejected_logps": -459.2474365234375,
      "epoch": 0.40625,
      "grad_norm": 4.398229659396608,
      "learning_rate": 1e-06,
      "logits/chosen": 1.508570909500122,
      "logits/rejected": 1.6251033544540405,
      "logps/chosen": -404.47314453125,
      "logps/rejected": -459.6101379394531,
      "loss": 0.4981,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.001276321243494749,
      "rewards/margins": 0.004903449676930904,
      "rewards/rejected": -0.0036271288990974426,
      "step": 13
    },
    {
      "debug/policy_chosen_logits": 1.7688783407211304,
      "debug/policy_chosen_logps": -424.601318359375,
      "debug/policy_rejected_logits": 1.3446720838546753,
      "debug/policy_rejected_logps": -452.760009765625,
      "debug/reference_chosen_logps": -423.881103515625,
      "debug/reference_rejected_logps": -452.08984375,
      "epoch": 0.4375,
      "grad_norm": 4.8812354492275505,
      "learning_rate": 1e-06,
      "logits/chosen": 1.7688783407211304,
      "logits/rejected": 1.3446720838546753,
      "logps/chosen": -424.601318359375,
      "logps/rejected": -452.760009765625,
      "loss": 0.5007,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.007202301174402237,
      "rewards/margins": -0.0005010606837458909,
      "rewards/rejected": -0.006701240316033363,
      "step": 14
    },
    {
      "debug/policy_chosen_logits": 1.4991490840911865,
      "debug/policy_chosen_logps": -414.95416259765625,
      "debug/policy_rejected_logits": 1.7444727420806885,
      "debug/policy_rejected_logps": -397.67449951171875,
      "debug/reference_chosen_logps": -414.56903076171875,
      "debug/reference_rejected_logps": -398.7281494140625,
      "epoch": 0.46875,
      "grad_norm": 4.614738630368307,
      "learning_rate": 1e-06,
      "logits/chosen": 1.4991490840911865,
      "logits/rejected": 1.7444727420806885,
      "logps/chosen": -414.95416259765625,
      "logps/rejected": -397.67449951171875,
      "loss": 0.5,
      "rewards/accuracies": 0.25,
      "rewards/chosen": -0.0038512798491865396,
      "rewards/margins": -0.014388121664524078,
      "rewards/rejected": 0.010536842048168182,
      "step": 15
    },
    {
      "debug/policy_chosen_logits": 1.8573402166366577,
      "debug/policy_chosen_logps": -411.7935485839844,
      "debug/policy_rejected_logits": 1.655826449394226,
      "debug/policy_rejected_logps": -428.7922668457031,
      "debug/reference_chosen_logps": -411.98040771484375,
      "debug/reference_rejected_logps": -428.9548645019531,
      "epoch": 0.5,
      "grad_norm": 4.37108878989768,
      "learning_rate": 1e-06,
      "logits/chosen": 1.8573402166366577,
      "logits/rejected": 1.655826449394226,
      "logps/chosen": -411.7935485839844,
      "logps/rejected": -428.7922668457031,
      "loss": 0.496,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0018686293624341488,
      "rewards/margins": 0.0002428237348794937,
      "rewards/rejected": 0.0016258049290627241,
      "step": 16
    },
    {
      "debug/policy_chosen_logits": 1.4710519313812256,
      "debug/policy_chosen_logps": -422.3893737792969,
      "debug/policy_rejected_logits": 1.6017435789108276,
      "debug/policy_rejected_logps": -444.07470703125,
      "debug/reference_chosen_logps": -421.87664794921875,
      "debug/reference_rejected_logps": -443.59698486328125,
      "epoch": 0.53125,
      "grad_norm": 3.9347513368146982,
      "learning_rate": 1e-06,
      "logits/chosen": 1.4710519313812256,
      "logits/rejected": 1.6017435789108276,
      "logps/chosen": -422.3893737792969,
      "logps/rejected": -444.07470703125,
      "loss": 0.4916,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.00512706721201539,
      "rewards/margins": -0.0003494260599836707,
      "rewards/rejected": -0.004777641035616398,
      "step": 17
    },
    {
      "debug/policy_chosen_logits": 1.6341142654418945,
      "debug/policy_chosen_logps": -445.0538330078125,
      "debug/policy_rejected_logits": 1.6374801397323608,
      "debug/policy_rejected_logps": -459.182373046875,
      "debug/reference_chosen_logps": -444.14422607421875,
      "debug/reference_rejected_logps": -458.5662841796875,
      "epoch": 0.5625,
      "grad_norm": 4.343729471925386,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6341142654418945,
      "logits/rejected": 1.6374801397323608,
      "logps/chosen": -445.0538330078125,
      "logps/rejected": -459.182373046875,
      "loss": 0.4974,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.00909606833010912,
      "rewards/margins": -0.0029354477301239967,
      "rewards/rejected": -0.006160621531307697,
      "step": 18
    },
    {
      "debug/policy_chosen_logits": 1.8724883794784546,
      "debug/policy_chosen_logps": -373.3547668457031,
      "debug/policy_rejected_logits": 1.7754302024841309,
      "debug/policy_rejected_logps": -470.7467956542969,
      "debug/reference_chosen_logps": -374.51361083984375,
      "debug/reference_rejected_logps": -470.7672119140625,
      "epoch": 0.59375,
      "grad_norm": 4.542983297185189,
      "learning_rate": 1e-06,
      "logits/chosen": 1.8724883794784546,
      "logits/rejected": 1.7754302024841309,
      "logps/chosen": -373.3547668457031,
      "logps/rejected": -470.7467956542969,
      "loss": 0.4987,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.011588573455810547,
      "rewards/margins": 0.011384601704776287,
      "rewards/rejected": 0.0002039717510342598,
      "step": 19
    },
    {
      "debug/policy_chosen_logits": 1.6738826036453247,
      "debug/policy_chosen_logps": -421.78070068359375,
      "debug/policy_rejected_logits": 1.6627225875854492,
      "debug/policy_rejected_logps": -457.22625732421875,
      "debug/reference_chosen_logps": -422.5181884765625,
      "debug/reference_rejected_logps": -456.66961669921875,
      "epoch": 0.625,
      "grad_norm": 4.212961782177093,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6738826036453247,
      "logits/rejected": 1.6627225875854492,
      "logps/chosen": -421.78070068359375,
      "logps/rejected": -457.22625732421875,
      "loss": 0.4966,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.007374763488769531,
      "rewards/margins": 0.012941397726535797,
      "rewards/rejected": -0.0055666351690888405,
      "step": 20
    },
    {
      "debug/policy_chosen_logits": 1.6218189001083374,
      "debug/policy_chosen_logps": -396.4649658203125,
      "debug/policy_rejected_logits": 1.6794122457504272,
      "debug/policy_rejected_logps": -393.181640625,
      "debug/reference_chosen_logps": -397.00018310546875,
      "debug/reference_rejected_logps": -393.26422119140625,
      "epoch": 0.65625,
      "grad_norm": 4.415514740564342,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6218189001083374,
      "logits/rejected": 1.6794122457504272,
      "logps/chosen": -396.4649658203125,
      "logps/rejected": -393.181640625,
      "loss": 0.4973,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0053519438952207565,
      "rewards/margins": 0.004526557866483927,
      "rewards/rejected": 0.0008253857959061861,
      "step": 21
    },
    {
      "debug/policy_chosen_logits": 1.8384886980056763,
      "debug/policy_chosen_logps": -445.74896240234375,
      "debug/policy_rejected_logits": 1.820858359336853,
      "debug/policy_rejected_logps": -427.93316650390625,
      "debug/reference_chosen_logps": -445.52197265625,
      "debug/reference_rejected_logps": -427.1968688964844,
      "epoch": 0.6875,
      "grad_norm": 4.279507111277066,
      "learning_rate": 1e-06,
      "logits/chosen": 1.8384886980056763,
      "logits/rejected": 1.820858359336853,
      "logps/chosen": -445.74896240234375,
      "logps/rejected": -427.93316650390625,
      "loss": 0.4976,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002269973512738943,
      "rewards/margins": 0.005093079060316086,
      "rewards/rejected": -0.007363053038716316,
      "step": 22
    },
    {
      "debug/policy_chosen_logits": 1.7056151628494263,
      "debug/policy_chosen_logps": -412.3712463378906,
      "debug/policy_rejected_logits": 1.6649104356765747,
      "debug/policy_rejected_logps": -451.2648620605469,
      "debug/reference_chosen_logps": -412.66668701171875,
      "debug/reference_rejected_logps": -449.33660888671875,
      "epoch": 0.71875,
      "grad_norm": 5.168198816512977,
      "learning_rate": 1e-06,
      "logits/chosen": 1.7056151628494263,
      "logits/rejected": 1.6649104356765747,
      "logps/chosen": -412.3712463378906,
      "logps/rejected": -451.2648620605469,
      "loss": 0.4922,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.00295448349788785,
      "rewards/margins": 0.022237051278352737,
      "rewards/rejected": -0.019282570108771324,
      "step": 23
    },
    {
      "debug/policy_chosen_logits": 1.3328857421875,
      "debug/policy_chosen_logps": -398.9266357421875,
      "debug/policy_rejected_logits": 1.627695083618164,
      "debug/policy_rejected_logps": -393.88433837890625,
      "debug/reference_chosen_logps": -399.486328125,
      "debug/reference_rejected_logps": -395.30322265625,
      "epoch": 0.75,
      "grad_norm": 5.635074846323111,
      "learning_rate": 1e-06,
      "logits/chosen": 1.3328857421875,
      "logits/rejected": 1.627695083618164,
      "logps/chosen": -398.9266357421875,
      "logps/rejected": -393.88433837890625,
      "loss": 0.5024,
      "rewards/accuracies": 0.25,
      "rewards/chosen": 0.005597000941634178,
      "rewards/margins": -0.00859165284782648,
      "rewards/rejected": 0.014188650995492935,
      "step": 24
    },
    {
      "debug/policy_chosen_logits": 1.1641021966934204,
      "debug/policy_chosen_logps": -426.21893310546875,
      "debug/policy_rejected_logits": 1.7478457689285278,
      "debug/policy_rejected_logps": -509.15692138671875,
      "debug/reference_chosen_logps": -426.0503845214844,
      "debug/reference_rejected_logps": -508.83636474609375,
      "epoch": 0.78125,
      "grad_norm": 4.992982311072277,
      "learning_rate": 1e-06,
      "logits/chosen": 1.1641021966934204,
      "logits/rejected": 1.7478457689285278,
      "logps/chosen": -426.21893310546875,
      "logps/rejected": -509.15692138671875,
      "loss": 0.4925,
      "rewards/accuracies": 0.5,
      "rewards/chosen": -0.00168533343821764,
      "rewards/margins": 0.0015203863149508834,
      "rewards/rejected": -0.003205719171091914,
      "step": 25
    },
    {
      "debug/policy_chosen_logits": 1.3662775754928589,
      "debug/policy_chosen_logps": -370.339111328125,
      "debug/policy_rejected_logits": 1.361480951309204,
      "debug/policy_rejected_logps": -431.66033935546875,
      "debug/reference_chosen_logps": -370.40283203125,
      "debug/reference_rejected_logps": -430.5782775878906,
      "epoch": 0.8125,
      "grad_norm": 5.426783468499563,
      "learning_rate": 1e-06,
      "logits/chosen": 1.3662775754928589,
      "logits/rejected": 1.361480951309204,
      "logps/chosen": -370.339111328125,
      "logps/rejected": -431.66033935546875,
      "loss": 0.4951,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0006370928604155779,
      "rewards/margins": 0.01145759504288435,
      "rewards/rejected": -0.01082050334662199,
      "step": 26
    },
    {
      "debug/policy_chosen_logits": 1.8342163562774658,
      "debug/policy_chosen_logps": -379.77557373046875,
      "debug/policy_rejected_logits": 1.710644245147705,
      "debug/policy_rejected_logps": -458.7286682128906,
      "debug/reference_chosen_logps": -381.13714599609375,
      "debug/reference_rejected_logps": -457.7099304199219,
      "epoch": 0.84375,
      "grad_norm": 4.18822192571096,
      "learning_rate": 1e-06,
      "logits/chosen": 1.8342163562774658,
      "logits/rejected": 1.710644245147705,
      "logps/chosen": -379.77557373046875,
      "logps/rejected": -458.7286682128906,
      "loss": 0.4942,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.013616141863167286,
      "rewards/margins": 0.02380317449569702,
      "rewards/rejected": -0.010187034495174885,
      "step": 27
    },
    {
      "debug/policy_chosen_logits": 1.6289498805999756,
      "debug/policy_chosen_logps": -342.238037109375,
      "debug/policy_rejected_logits": 1.474778413772583,
      "debug/policy_rejected_logps": -447.5248718261719,
      "debug/reference_chosen_logps": -343.45904541015625,
      "debug/reference_rejected_logps": -446.7528076171875,
      "epoch": 0.875,
      "grad_norm": 4.120783195357869,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6289498805999756,
      "logits/rejected": 1.474778413772583,
      "logps/chosen": -342.238037109375,
      "logps/rejected": -447.5248718261719,
      "loss": 0.4991,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.012210273183882236,
      "rewards/margins": 0.019930992275476456,
      "rewards/rejected": -0.007720718160271645,
      "step": 28
    },
    {
      "debug/policy_chosen_logits": 1.6164311170578003,
      "debug/policy_chosen_logps": -397.0234680175781,
      "debug/policy_rejected_logits": 1.5657469034194946,
      "debug/policy_rejected_logps": -414.18756103515625,
      "debug/reference_chosen_logps": -397.58160400390625,
      "debug/reference_rejected_logps": -414.6282958984375,
      "epoch": 0.90625,
      "grad_norm": 4.352593213110272,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6164311170578003,
      "logits/rejected": 1.5657469034194946,
      "logps/chosen": -397.0234680175781,
      "logps/rejected": -414.18756103515625,
      "loss": 0.4981,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.005581206642091274,
      "rewards/margins": 0.0011737439781427383,
      "rewards/rejected": 0.004407462663948536,
      "step": 29
    },
    {
      "debug/policy_chosen_logits": 1.3324956893920898,
      "debug/policy_chosen_logps": -384.20556640625,
      "debug/policy_rejected_logits": 1.4190467596054077,
      "debug/policy_rejected_logps": -439.9374694824219,
      "debug/reference_chosen_logps": -384.3076171875,
      "debug/reference_rejected_logps": -439.364990234375,
      "epoch": 0.9375,
      "grad_norm": 4.24197252932316,
      "learning_rate": 1e-06,
      "logits/chosen": 1.3324956893920898,
      "logits/rejected": 1.4190467596054077,
      "logps/chosen": -384.20556640625,
      "logps/rejected": -439.9374694824219,
      "loss": 0.4902,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0010205453727394342,
      "rewards/margins": 0.006745452992618084,
      "rewards/rejected": -0.005724907387048006,
      "step": 30
    },
    {
      "debug/policy_chosen_logits": 1.6599881649017334,
      "debug/policy_chosen_logps": -391.8392333984375,
      "debug/policy_rejected_logits": 1.440269947052002,
      "debug/policy_rejected_logps": -410.36962890625,
      "debug/reference_chosen_logps": -391.9766845703125,
      "debug/reference_rejected_logps": -410.27911376953125,
      "epoch": 0.96875,
      "grad_norm": 4.2333369790689614,
      "learning_rate": 1e-06,
      "logits/chosen": 1.6599881649017334,
      "logits/rejected": 1.440269947052002,
      "logps/chosen": -391.8392333984375,
      "logps/rejected": -410.36962890625,
      "loss": 0.4915,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.0013742828741669655,
      "rewards/margins": 0.002279548905789852,
      "rewards/rejected": -0.0009052654495462775,
      "step": 31
    },
    {
      "debug/policy_chosen_logits": 1.3013134002685547,
      "debug/policy_chosen_logps": -468.1767883300781,
      "debug/policy_rejected_logits": 1.5036485195159912,
      "debug/policy_rejected_logps": -528.7972412109375,
      "debug/reference_chosen_logps": -468.2105712890625,
      "debug/reference_rejected_logps": -525.7525634765625,
      "epoch": 1.0,
      "grad_norm": 4.709201002942774,
      "learning_rate": 1e-06,
      "logits/chosen": 1.3013134002685547,
      "logits/rejected": 1.5036485195159912,
      "logps/chosen": -468.1767883300781,
      "logps/rejected": -528.7972412109375,
      "loss": 0.4547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.00033790571615099907,
      "rewards/margins": 0.03078407421708107,
      "rewards/rejected": -0.030446168035268784,
      "step": 32
    },
    {
      "epoch": 1.0,
      "step": 32,
      "total_flos": 0.0,
      "train_loss": 0.49622934125363827,
      "train_runtime": 397.2698,
      "train_samples_per_second": 5.022,
      "train_steps_per_second": 0.081
    }
  ],
  "logging_steps": 1,
  "max_steps": 32,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 8,
  "trial_name": null,
  "trial_params": null
}