End of training

README.md

@@ -2,11 +2,23 @@
 library_name: transformers
 tags:
 - generated_from_trainer
+datasets:
+- kanishka/babylm2-rewritten-clean-spacy
 metrics:
 - accuracy
 model-index:
 - name: opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-211_1e-3
-  results: []
+  results:
+  - task:
+      name: Causal Language Modeling
+      type: text-generation
+    dataset:
+      name: kanishka/babylm2-rewritten-clean-spacy
+      type: kanishka/babylm2-rewritten-clean-spacy
+    metrics:
+    - name: Accuracy
+      type: accuracy
+      value: 0.47886644176457666
 ---
 
 <!-- This model card has been generated automatically according to the information the Trainer had access to. You
@@ -14,7 +26,7 @@ should probably proofread and complete it, then remove this comment. -->
 
 # opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-211_1e-3
 
-This model was trained from scratch on an unknown dataset.
+This model was trained from scratch on the kanishka/babylm2-rewritten-clean-spacy dataset.
 It achieves the following results on the evaluation set:
 - Loss: 2.6837
 - Accuracy: 0.4789

all_results.json

@@ -0,0 +1,16 @@
+{
+    "epoch": 19.991464360935595,
+    "eval_accuracy": 0.47886644176457666,
+    "eval_loss": 2.683746337890625,
+    "eval_runtime": 71.9708,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 843.12,
+    "eval_steps_per_second": 13.186,
+    "perplexity": 14.639836462706617,
+    "total_flos": 1.507910045663232e+18,
+    "train_loss": 2.80528530789056,
+    "train_runtime": 30473.0676,
+    "train_samples": 577344,
+    "train_samples_per_second": 378.921,
+    "train_steps_per_second": 1.48
+}

eval_results.json

@@ -0,0 +1,10 @@
+{
+    "epoch": 19.991464360935595,
+    "eval_accuracy": 0.47886644176457666,
+    "eval_loss": 2.683746337890625,
+    "eval_runtime": 71.9708,
+    "eval_samples": 60680,
+    "eval_samples_per_second": 843.12,
+    "eval_steps_per_second": 13.186,
+    "perplexity": 14.639836462706617
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 19.991464360935595,
+    "total_flos": 1.507910045663232e+18,
+    "train_loss": 2.80528530789056,
+    "train_runtime": 30473.0676,
+    "train_samples": 577344,
+    "train_samples_per_second": 378.921,
+    "train_steps_per_second": 1.48
+}

trainer_state.json

@@ -0,0 +1,546 @@
+{
+  "best_metric": 2.683746337890625,
+  "best_model_checkpoint": "models/opt-babylm2-rewritten-clean-spacy-earlystop-bpe_seed-211_1e-3/checkpoint-45100",
+  "epoch": 19.991464360935595,
+  "eval_steps": 500,
+  "global_step": 45100,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.4434098215275468,
+      "grad_norm": 0.535893440246582,
+      "learning_rate": 3.125e-05,
+      "loss": 5.5759,
+      "step": 1000
+    },
+    {
+      "epoch": 0.8868196430550936,
+      "grad_norm": 0.5292273163795471,
+      "learning_rate": 6.25e-05,
+      "loss": 4.1,
+      "step": 2000
+    },
+    {
+      "epoch": 1.0,
+      "eval_accuracy": 0.36012162808432535,
+      "eval_loss": 3.8208348751068115,
+      "eval_runtime": 73.9931,
+      "eval_samples_per_second": 820.076,
+      "eval_steps_per_second": 12.826,
+      "step": 2256
+    },
+    {
+      "epoch": 1.3298969072164948,
+      "grad_norm": 0.6377725601196289,
+      "learning_rate": 9.375e-05,
+      "loss": 3.7036,
+      "step": 3000
+    },
+    {
+      "epoch": 1.7733067287440418,
+      "grad_norm": 0.5576291680335999,
+      "learning_rate": 0.000125,
+      "loss": 3.4569,
+      "step": 4000
+    },
+    {
+      "epoch": 2.0,
+      "eval_accuracy": 0.40816362273320667,
+      "eval_loss": 3.308465003967285,
+      "eval_runtime": 73.782,
+      "eval_samples_per_second": 822.423,
+      "eval_steps_per_second": 12.862,
+      "step": 4512
+    },
+    {
+      "epoch": 2.2163839929054427,
+      "grad_norm": 0.5923783183097839,
+      "learning_rate": 0.00015625,
+      "loss": 3.2553,
+      "step": 5000
+    },
+    {
+      "epoch": 2.6597938144329896,
+      "grad_norm": 0.4800017178058624,
+      "learning_rate": 0.0001875,
+      "loss": 3.1304,
+      "step": 6000
+    },
+    {
+      "epoch": 3.0,
+      "eval_accuracy": 0.4293314979254721,
+      "eval_loss": 3.0989913940429688,
+      "eval_runtime": 73.5868,
+      "eval_samples_per_second": 824.605,
+      "eval_steps_per_second": 12.896,
+      "step": 6768
+    },
+    {
+      "epoch": 3.102871078594391,
+      "grad_norm": 0.40440040826797485,
+      "learning_rate": 0.00021875,
+      "loss": 3.0393,
+      "step": 7000
+    },
+    {
+      "epoch": 3.5462809001219378,
+      "grad_norm": 0.3914831876754761,
+      "learning_rate": 0.00025,
+      "loss": 2.9686,
+      "step": 8000
+    },
+    {
+      "epoch": 3.9896907216494846,
+      "grad_norm": 0.37147632241249084,
+      "learning_rate": 0.00028125000000000003,
+      "loss": 2.9241,
+      "step": 9000
+    },
+    {
+      "epoch": 4.0,
+      "eval_accuracy": 0.44051643465560253,
+      "eval_loss": 2.9892077445983887,
+      "eval_runtime": 73.1132,
+      "eval_samples_per_second": 829.946,
+      "eval_steps_per_second": 12.98,
+      "step": 9024
+    },
+    {
+      "epoch": 4.4327679858108855,
+      "grad_norm": 0.3617306649684906,
+      "learning_rate": 0.0003125,
+      "loss": 2.8599,
+      "step": 10000
+    },
+    {
+      "epoch": 4.876177807338433,
+      "grad_norm": 0.31954342126846313,
+      "learning_rate": 0.00034375,
+      "loss": 2.8456,
+      "step": 11000
+    },
+    {
+      "epoch": 5.0,
+      "eval_accuracy": 0.44670376258611555,
+      "eval_loss": 2.9302828311920166,
+      "eval_runtime": 72.5611,
+      "eval_samples_per_second": 836.261,
+      "eval_steps_per_second": 13.079,
+      "step": 11280
+    },
+    {
+      "epoch": 5.319255071499834,
+      "grad_norm": 0.32251182198524475,
+      "learning_rate": 0.000375,
+      "loss": 2.7976,
+      "step": 12000
+    },
+    {
+      "epoch": 5.762664893027381,
+      "grad_norm": 0.312112957239151,
+      "learning_rate": 0.00040625000000000004,
+      "loss": 2.7882,
+      "step": 13000
+    },
+    {
+      "epoch": 6.0,
+      "eval_accuracy": 0.4509860793361511,
+      "eval_loss": 2.888702869415283,
+      "eval_runtime": 72.4551,
+      "eval_samples_per_second": 837.484,
+      "eval_steps_per_second": 13.098,
+      "step": 13536
+    },
+    {
+      "epoch": 6.205742157188782,
+      "grad_norm": 0.28876274824142456,
+      "learning_rate": 0.0004375,
+      "loss": 2.7559,
+      "step": 14000
+    },
+    {
+      "epoch": 6.649151978716328,
+      "grad_norm": 0.29188501834869385,
+      "learning_rate": 0.00046875,
+      "loss": 2.7426,
+      "step": 15000
+    },
+    {
+      "epoch": 7.0,
+      "eval_accuracy": 0.4542816058526245,
+      "eval_loss": 2.8618035316467285,
+      "eval_runtime": 72.3378,
+      "eval_samples_per_second": 838.842,
+      "eval_steps_per_second": 13.119,
+      "step": 15792
+    },
+    {
+      "epoch": 7.09222924287773,
+      "grad_norm": 0.28137603402137756,
+      "learning_rate": 0.0005,
+      "loss": 2.7305,
+      "step": 16000
+    },
+    {
+      "epoch": 7.535639064405276,
+      "grad_norm": 0.25505542755126953,
+      "learning_rate": 0.00053121875,
+      "loss": 2.707,
+      "step": 17000
+    },
+    {
+      "epoch": 7.979048885932824,
+      "grad_norm": 0.24025394022464752,
+      "learning_rate": 0.0005624687499999999,
+      "loss": 2.7124,
+      "step": 18000
+    },
+    {
+      "epoch": 8.0,
+      "eval_accuracy": 0.4565116264040224,
+      "eval_loss": 2.840358257293701,
+      "eval_runtime": 72.5997,
+      "eval_samples_per_second": 835.816,
+      "eval_steps_per_second": 13.072,
+      "step": 18048
+    },
+    {
+      "epoch": 8.422126150094225,
+      "grad_norm": 0.24389903247356415,
+      "learning_rate": 0.0005936875,
+      "loss": 2.6715,
+      "step": 19000
+    },
+    {
+      "epoch": 8.865535971621771,
+      "grad_norm": 0.23571985960006714,
+      "learning_rate": 0.0006249375000000001,
+      "loss": 2.689,
+      "step": 20000
+    },
+    {
+      "epoch": 9.0,
+      "eval_accuracy": 0.45795106440730543,
+      "eval_loss": 2.8292860984802246,
+      "eval_runtime": 72.5327,
+      "eval_samples_per_second": 836.588,
+      "eval_steps_per_second": 13.084,
+      "step": 20304
+    },
+    {
+      "epoch": 9.308613235783172,
+      "grad_norm": 0.2347807139158249,
+      "learning_rate": 0.0006561562500000001,
+      "loss": 2.6603,
+      "step": 21000
+    },
+    {
+      "epoch": 9.75202305731072,
+      "grad_norm": 0.21437448263168335,
+      "learning_rate": 0.00068734375,
+      "loss": 2.6674,
+      "step": 22000
+    },
+    {
+      "epoch": 10.0,
+      "eval_accuracy": 0.45918653948065713,
+      "eval_loss": 2.817204236984253,
+      "eval_runtime": 72.5446,
+      "eval_samples_per_second": 836.45,
+      "eval_steps_per_second": 13.082,
+      "step": 22560
+    },
+    {
+      "epoch": 10.19510032147212,
+      "grad_norm": 0.21189159154891968,
+      "learning_rate": 0.00071859375,
+      "loss": 2.6505,
+      "step": 23000
+    },
+    {
+      "epoch": 10.638510142999667,
+      "grad_norm": 0.20313581824302673,
+      "learning_rate": 0.0007498437500000001,
+      "loss": 2.651,
+      "step": 24000
+    },
+    {
+      "epoch": 11.0,
+      "eval_accuracy": 0.4604101231791332,
+      "eval_loss": 2.8066623210906982,
+      "eval_runtime": 72.4079,
+      "eval_samples_per_second": 838.03,
+      "eval_steps_per_second": 13.106,
+      "step": 24816
+    },
+    {
+      "epoch": 11.081587407161068,
+      "grad_norm": 0.20089128613471985,
+      "learning_rate": 0.00078109375,
+      "loss": 2.6463,
+      "step": 25000
+    },
+    {
+      "epoch": 11.524997228688616,
+      "grad_norm": 0.20063355565071106,
+      "learning_rate": 0.0008123125,
+      "loss": 2.633,
+      "step": 26000
+    },
+    {
+      "epoch": 11.968407050216163,
+      "grad_norm": 0.18298469483852386,
+      "learning_rate": 0.0008435625,
+      "loss": 2.6483,
+      "step": 27000
+    },
+    {
+      "epoch": 12.0,
+      "eval_accuracy": 0.4612719247224269,
+      "eval_loss": 2.800128221511841,
+      "eval_runtime": 72.4998,
+      "eval_samples_per_second": 836.967,
+      "eval_steps_per_second": 13.09,
+      "step": 27072
+    },
+    {
+      "epoch": 12.411484314377564,
+      "grad_norm": 0.19970569014549255,
+      "learning_rate": 0.00087478125,
+      "loss": 2.6154,
+      "step": 28000
+    },
+    {
+      "epoch": 12.854894135905111,
+      "grad_norm": 0.17127935588359833,
+      "learning_rate": 0.0009060312499999999,
+      "loss": 2.639,
+      "step": 29000
+    },
+    {
+      "epoch": 13.0,
+      "eval_accuracy": 0.46190145669342225,
+      "eval_loss": 2.7980594635009766,
+      "eval_runtime": 72.3226,
+      "eval_samples_per_second": 839.018,
+      "eval_steps_per_second": 13.122,
+      "step": 29328
+    },
+    {
+      "epoch": 13.297971400066512,
+      "grad_norm": 0.19391290843486786,
+      "learning_rate": 0.00093725,
+      "loss": 2.6128,
+      "step": 30000
+    },
+    {
+      "epoch": 13.741381221594057,
+      "grad_norm": 0.1826080083847046,
+      "learning_rate": 0.0009685000000000001,
+      "loss": 2.6268,
+      "step": 31000
+    },
+    {
+      "epoch": 14.0,
+      "eval_accuracy": 0.46204221438080834,
+      "eval_loss": 2.7923851013183594,
+      "eval_runtime": 72.136,
+      "eval_samples_per_second": 841.189,
+      "eval_steps_per_second": 13.156,
+      "step": 31584
+    },
+    {
+      "epoch": 14.18445848575546,
+      "grad_norm": 0.1709497720003128,
+      "learning_rate": 0.00099971875,
+      "loss": 2.6164,
+      "step": 32000
+    },
+    {
+      "epoch": 14.627868307283006,
+      "grad_norm": 0.1697809398174286,
+      "learning_rate": 0.000924351145038168,
+      "loss": 2.6116,
+      "step": 33000
+    },
+    {
+      "epoch": 15.0,
+      "eval_accuracy": 0.46505868134992956,
+      "eval_loss": 2.766456365585327,
+      "eval_runtime": 72.5226,
+      "eval_samples_per_second": 836.705,
+      "eval_steps_per_second": 13.086,
+      "step": 33840
+    },
+    {
+      "epoch": 15.070945571444408,
+      "grad_norm": 0.1920733004808426,
+      "learning_rate": 0.0008480152671755725,
+      "loss": 2.5972,
+      "step": 34000
+    },
+    {
+      "epoch": 15.514355392971954,
+      "grad_norm": 0.17531998455524445,
+      "learning_rate": 0.0007717557251908397,
+      "loss": 2.5637,
+      "step": 35000
+    },
+    {
+      "epoch": 15.957765214499501,
+      "grad_norm": 0.175877183675766,
+      "learning_rate": 0.0006954198473282442,
+      "loss": 2.5657,
+      "step": 36000
+    },
+    {
+      "epoch": 16.0,
+      "eval_accuracy": 0.4687057789496814,
+      "eval_loss": 2.7436182498931885,
+      "eval_runtime": 72.9943,
+      "eval_samples_per_second": 831.298,
+      "eval_steps_per_second": 13.001,
+      "step": 36096
+    },
+    {
+      "epoch": 16.400842478660902,
+      "grad_norm": 0.18140995502471924,
+      "learning_rate": 0.0006191603053435114,
+      "loss": 2.509,
+      "step": 37000
+    },
+    {
+      "epoch": 16.84425230018845,
+      "grad_norm": 0.17342546582221985,
+      "learning_rate": 0.0005428244274809161,
+      "loss": 2.5199,
+      "step": 38000
+    },
+    {
+      "epoch": 17.0,
+      "eval_accuracy": 0.47174531777114276,
+      "eval_loss": 2.7186553478240967,
+      "eval_runtime": 72.9122,
+      "eval_samples_per_second": 832.234,
+      "eval_steps_per_second": 13.016,
+      "step": 38352
+    },
+    {
+      "epoch": 17.28732956434985,
+      "grad_norm": 0.18188068270683289,
+      "learning_rate": 0.0004665648854961832,
+      "loss": 2.4714,
+      "step": 39000
+    },
+    {
+      "epoch": 17.730739385877396,
+      "grad_norm": 0.18511821329593658,
+      "learning_rate": 0.0003902290076335878,
+      "loss": 2.4626,
+      "step": 40000
+    },
+    {
+      "epoch": 18.0,
+      "eval_accuracy": 0.4746070676128065,
+      "eval_loss": 2.7018399238586426,
+      "eval_runtime": 72.5891,
+      "eval_samples_per_second": 835.938,
+      "eval_steps_per_second": 13.074,
+      "step": 40608
+    },
+    {
+      "epoch": 18.1738166500388,
+      "grad_norm": 0.18930885195732117,
+      "learning_rate": 0.0003138931297709924,
+      "loss": 2.4292,
+      "step": 41000
+    },
+    {
+      "epoch": 18.617226471566344,
+      "grad_norm": 0.19555842876434326,
+      "learning_rate": 0.00023763358778625953,
+      "loss": 2.3982,
+      "step": 42000
+    },
+    {
+      "epoch": 19.0,
+      "eval_accuracy": 0.47732812439412153,
+      "eval_loss": 2.6869590282440186,
+      "eval_runtime": 72.705,
+      "eval_samples_per_second": 834.605,
+      "eval_steps_per_second": 13.053,
+      "step": 42864
+    },
+    {
+      "epoch": 19.060303735727746,
+      "grad_norm": 0.2038242667913437,
+      "learning_rate": 0.00016129770992366412,
+      "loss": 2.3835,
+      "step": 43000
+    },
+    {
+      "epoch": 19.503713557255292,
+      "grad_norm": 0.20078131556510925,
+      "learning_rate": 8.50381679389313e-05,
+      "loss": 2.3292,
+      "step": 44000
+    },
+    {
+      "epoch": 19.94712337878284,
+      "grad_norm": 0.19901303946971893,
+      "learning_rate": 8.702290076335879e-06,
+      "loss": 2.3238,
+      "step": 45000
+    },
+    {
+      "epoch": 19.991464360935595,
+      "eval_accuracy": 0.47886644176457666,
+      "eval_loss": 2.683746337890625,
+      "eval_runtime": 72.7112,
+      "eval_samples_per_second": 834.534,
+      "eval_steps_per_second": 13.052,
+      "step": 45100
+    },
+    {
+      "epoch": 19.991464360935595,
+      "step": 45100,
+      "total_flos": 1.507910045663232e+18,
+      "train_loss": 2.80528530789056,
+      "train_runtime": 30473.0676,
+      "train_samples_per_second": 378.921,
+      "train_steps_per_second": 1.48
+    }
+  ],
+  "logging_steps": 1000,
+  "max_steps": 45100,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 20,
+  "save_steps": 500,
+  "stateful_callbacks": {
+    "EarlyStoppingCallback": {
+      "args": {
+        "early_stopping_patience": 3,
+        "early_stopping_threshold": 0.0
+      },
+      "attributes": {
+        "early_stopping_patience_counter": 0
+      }
+    },
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 1.507910045663232e+18,
+  "train_batch_size": 32,
+  "trial_name": null,
+  "trial_params": null
+}