Upload with huggingface_hub

added_tokens.json

@@ -1,66 +1,3 @@
 {
-  "<pad>": 32000,
-  "<pad_0>": 32001,
-  "<pad_10>": 32011,
-  "<pad_11>": 32012,
-  "<pad_12>": 32013,
-  "<pad_13>": 32014,
-  "<pad_14>": 32015,
-  "<pad_15>": 32016,
-  "<pad_16>": 32017,
-  "<pad_17>": 32018,
-  "<pad_18>": 32019,
-  "<pad_19>": 32020,
-  "<pad_1>": 32002,
-  "<pad_20>": 32021,
-  "<pad_21>": 32022,
-  "<pad_22>": 32023,
-  "<pad_23>": 32024,
-  "<pad_24>": 32025,
-  "<pad_25>": 32026,
-  "<pad_26>": 32027,
-  "<pad_27>": 32028,
-  "<pad_28>": 32029,
-  "<pad_29>": 32030,
-  "<pad_2>": 32003,
-  "<pad_30>": 32031,
-  "<pad_31>": 32032,
-  "<pad_32>": 32033,
-  "<pad_33>": 32034,
-  "<pad_34>": 32035,
-  "<pad_35>": 32036,
-  "<pad_36>": 32037,
-  "<pad_37>": 32038,
-  "<pad_38>": 32039,
-  "<pad_39>": 32040,
-  "<pad_3>": 32004,
-  "<pad_40>": 32041,
-  "<pad_41>": 32042,
-  "<pad_42>": 32043,
-  "<pad_43>": 32044,
-  "<pad_44>": 32045,
-  "<pad_45>": 32046,
-  "<pad_46>": 32047,
-  "<pad_47>": 32048,
-  "<pad_48>": 32049,
-  "<pad_49>": 32050,
-  "<pad_4>": 32005,
-  "<pad_50>": 32051,
-  "<pad_51>": 32052,
-  "<pad_52>": 32053,
-  "<pad_53>": 32054,
-  "<pad_54>": 32055,
-  "<pad_55>": 32056,
-  "<pad_56>": 32057,
-  "<pad_57>": 32058,
-  "<pad_58>": 32059,
-  "<pad_59>": 32060,
-  "<pad_5>": 32006,
-  "<pad_60>": 32061,
-  "<pad_61>": 32062,
-  "<pad_62>": 32063,
-  "<pad_6>": 32007,
-  "<pad_7>": 32008,
-  "<pad_8>": 32009,
-  "<pad_9>": 32010
+  "<pad>": 32000
 }

config.json

@@ -1,5 +1,5 @@
 {
-  "_name_or_path": "swype/deepshard-13B-ft",
+  "_name_or_path": "swype/deepshard-13B-raw",
   "architectures": [
     "LlamaForCausalLM"
   ],
@@ -9,7 +9,6 @@
   "hidden_size": 5120,
   "initializer_range": 0.02,
   "intermediate_size": 13824,
-  "max_sequence_length": 2048,
   "model_type": "llama",
   "num_attention_heads": 40,
   "num_hidden_layers": 40,
@@ -19,5 +18,5 @@
   "torch_dtype": "float32",
   "transformers_version": "4.28.0.dev0",
   "use_cache": true,
-  "vocab_size": 32064
+  "vocab_size": 32001
 }

pytorch_model-00001-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b471e38cf7666d29c05d5a2608612cac6025986831016dab57ae0d6894345622
-size 9957854603
+oid sha256:c79e83b4b6e934facb51f76de7425701fd53520793fc3a62ee6f6e986d1dfd38
+size 9956564363

pytorch_model-00002-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:4851b0457ff807013f6d89e0464541d2dc6c2ddcde9b4b0cd4ae60a0ca681095
+oid sha256:54376ff86e52e85da8064236d2420e489ddd80ee5852c3c671d7211e0a3d9629
 size 9940856385

pytorch_model-00003-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:69d6760872e9a7660c51e7ed93f586277e0139d18d3997e6c10a235bd86c04c3
+oid sha256:a4a3294d4ae14e52260acbd6fba427c21e4729fd6b4948f0748b535924d51209
 size 9940856943

pytorch_model-00004-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6a3798c9896eb6f3408dd7747252ef8df7b22fab32a6fd994181af393abc5d1
+oid sha256:cf196ac8e5dfa0e9b478f71b54bfbd050931242a20f37bcf6f550fc0e5c9ff0b
 size 9867415289

pytorch_model-00005-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:57748377c4aaa8a8cbd182d7a96593289b698b895c664746919708ab30482ff3
+oid sha256:0b7113f68e1a8cd3b1ce01926f3e6b219300ac3ed226c25eb7ab4069af8131c1
 size 9867456961

pytorch_model-00006-of-00006.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:5de63b0309f9c2f2dec60e21f88f0c02c9afc3fcddd4056f480836a6f23ce29e
-size 2491786927
+oid sha256:a7860cdc7e83a7380f546a53c6382eca2d41291b0f1ace9bd392c2d5b39d2d9c
+size 2490496687

pytorch_model.bin.index.json

@@ -1,6 +1,6 @@
 {
   "metadata": {
-    "total_size": 52066088960
+    "total_size": 52063508480
   },
   "weight_map": {
     "lm_head.weight": "pytorch_model-00006-of-00006.bin",

runs/Mar22_22-47-46_pslw9jwq6/1679525384.1219966/events.out.tfevents.1679525384.pslw9jwq6.121621.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a97748f7318111b3a504003d814ecb0eb5e9c829ab92ee26fde7e83b53081b0
+size 6071

runs/Mar22_22-47-46_pslw9jwq6/events.out.tfevents.1679525384.pslw9jwq6.121621.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fcb96b622ee819abfbfb30e3669801b1443190effcd18e3719f5e42affc89ba
+size 4893

special_tokens_map.json

@@ -1,68 +1,6 @@
 {
-  "additional_special_tokens": [
-    "<pad_0>",
-    "<pad_1>",
-    "<pad_2>",
-    "<pad_3>",
-    "<pad_4>",
-    "<pad_5>",
-    "<pad_6>",
-    "<pad_7>",
-    "<pad_8>",
-    "<pad_9>",
-    "<pad_10>",
-    "<pad_11>",
-    "<pad_12>",
-    "<pad_13>",
-    "<pad_14>",
-    "<pad_15>",
-    "<pad_16>",
-    "<pad_17>",
-    "<pad_18>",
-    "<pad_19>",
-    "<pad_20>",
-    "<pad_21>",
-    "<pad_22>",
-    "<pad_23>",
-    "<pad_24>",
-    "<pad_25>",
-    "<pad_26>",
-    "<pad_27>",
-    "<pad_28>",
-    "<pad_29>",
-    "<pad_30>",
-    "<pad_31>",
-    "<pad_32>",
-    "<pad_33>",
-    "<pad_34>",
-    "<pad_35>",
-    "<pad_36>",
-    "<pad_37>",
-    "<pad_38>",
-    "<pad_39>",
-    "<pad_40>",
-    "<pad_41>",
-    "<pad_42>",
-    "<pad_43>",
-    "<pad_44>",
-    "<pad_45>",
-    "<pad_46>",
-    "<pad_47>",
-    "<pad_48>",
-    "<pad_49>",
-    "<pad_50>",
-    "<pad_51>",
-    "<pad_52>",
-    "<pad_53>",
-    "<pad_54>",
-    "<pad_55>",
-    "<pad_56>",
-    "<pad_57>",
-    "<pad_58>",
-    "<pad_59>",
-    "<pad_60>",
-    "<pad_61>",
-    "<pad_62>"
-  ],
-  "pad_token": "<pad_0>"
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "<pad>",
+  "unk_token": "<unk>"
 }

tokenizer_config.json

@@ -1,9 +1,9 @@
 {
-  "bos_token": "",
-  "eos_token": "",
+  "bos_token": "<s>",
+  "eos_token": "</s>",
   "model_max_length": 512,
   "padding_side": "right",
-  "special_tokens_map_file": "/home/ubuntu/.cache/huggingface/hub/models--swype--deepshard-13B-raw/snapshots/225c31ea4b7db4ec9277178b37bb8fbea50aa258/special_tokens_map.json",
+  "special_tokens_map_file": "/home/paperspace/.cache/huggingface/hub/models--swype--deepshard-13B-raw/snapshots/956dd7d2c805d149e9c3d4bef1c16c2df6802408/special_tokens_map.json",
   "tokenizer_class": "LlamaTokenizer",
-  "unk_token": ""
+  "unk_token": "<unk>"
 }

trainer_state.json

@@ -2,266 +2,42 @@
   "best_metric": null,
   "best_model_checkpoint": null,
   "epoch": 3.0,
-  "global_step": 39,
+  "global_step": 3,
   "is_hyper_param_search": false,
   "is_local_process_zero": true,
   "is_world_process_zero": true,
   "log_history": [
-    {
-      "epoch": 0.08,
-      "learning_rate": 1e-05,
-      "loss": 0.5431,
-      "step": 1
-    },
-    {
-      "epoch": 0.15,
-      "learning_rate": 2e-05,
-      "loss": 0.5217,
-      "step": 2
-    },
-    {
-      "epoch": 0.23,
-      "learning_rate": 1.9963974885425267e-05,
-      "loss": 0.4643,
-      "step": 3
-    },
-    {
-      "epoch": 0.31,
-      "learning_rate": 1.9856159103477085e-05,
-      "loss": 0.4449,
-      "step": 4
-    },
-    {
-      "epoch": 0.38,
-      "learning_rate": 1.967732946933499e-05,
-      "loss": 0.4399,
-      "step": 5
-    },
-    {
-      "epoch": 0.46,
-      "learning_rate": 1.9428774454610845e-05,
-      "loss": 0.4314,
-      "step": 6
-    },
-    {
-      "epoch": 0.54,
-      "learning_rate": 1.911228490388136e-05,
-      "loss": 0.44,
-      "step": 7
-    },
-    {
-      "epoch": 0.62,
-      "learning_rate": 1.8730141131611882e-05,
-      "loss": 0.422,
-      "step": 8
-    },
-    {
-      "epoch": 0.69,
-      "learning_rate": 1.8285096492438424e-05,
-      "loss": 0.4005,
-      "step": 9
-    },
-    {
-      "epoch": 0.77,
-      "learning_rate": 1.7780357543184396e-05,
-      "loss": 0.4117,
-      "step": 10
-    },
-    {
-      "epoch": 0.85,
-      "learning_rate": 1.7219560939545246e-05,
-      "loss": 0.3954,
-      "step": 11
-    },
-    {
-      "epoch": 0.92,
-      "learning_rate": 1.6606747233900816e-05,
-      "loss": 0.3866,
-      "step": 12
-    },
     {
       "epoch": 1.0,
-      "learning_rate": 1.594633176304287e-05,
-      "loss": 0.3906,
-      "step": 13
-    },
-    {
-      "epoch": 1.08,
-      "learning_rate": 1.5243072835572319e-05,
-      "loss": 0.3271,
-      "step": 14
-    },
-    {
-      "epoch": 1.15,
-      "learning_rate": 1.4502037448176734e-05,
-      "loss": 0.3647,
-      "step": 15
-    },
-    {
-      "epoch": 1.23,
-      "learning_rate": 1.3728564777803089e-05,
-      "loss": 0.3168,
-      "step": 16
-    },
-    {
-      "epoch": 1.31,
-      "learning_rate": 1.2928227712765504e-05,
-      "loss": 0.3172,
-      "step": 17
-    },
-    {
-      "epoch": 1.38,
-      "learning_rate": 1.2106792699957264e-05,
-      "loss": 0.3152,
-      "step": 18
-    },
-    {
-      "epoch": 1.46,
-      "learning_rate": 1.1270178197468788e-05,
-      "loss": 0.3392,
-      "step": 19
-    },
-    {
-      "epoch": 1.54,
-      "learning_rate": 1.0424412031961485e-05,
-      "loss": 0.3301,
-      "step": 20
-    },
-    {
-      "epoch": 1.62,
-      "learning_rate": 9.57558796803852e-06,
-      "loss": 0.2919,
-      "step": 21
-    },
-    {
-      "epoch": 1.69,
-      "learning_rate": 8.729821802531213e-06,
-      "loss": 0.2873,
-      "step": 22
-    },
-    {
-      "epoch": 1.77,
-      "learning_rate": 7.89320730004274e-06,
-      "loss": 0.3345,
-      "step": 23
-    },
-    {
-      "epoch": 1.85,
-      "learning_rate": 7.071772287234497e-06,
-      "loss": 0.3276,
-      "step": 24
-    },
-    {
-      "epoch": 1.92,
-      "learning_rate": 6.2714352221969155e-06,
-      "loss": 0.2866,
-      "step": 25
+      "learning_rate": 2e-05,
+      "loss": 1.0501,
+      "step": 1
     },
     {
       "epoch": 2.0,
-      "learning_rate": 5.497962551823266e-06,
-      "loss": 0.2947,
-      "step": 26
-    },
-    {
-      "epoch": 2.08,
-      "learning_rate": 4.756927164427685e-06,
-      "loss": 0.2738,
-      "step": 27
-    },
-    {
-      "epoch": 2.15,
-      "learning_rate": 4.053668236957135e-06,
-      "loss": 0.2464,
-      "step": 28
-    },
-    {
-      "epoch": 2.23,
-      "learning_rate": 3.3932527660991877e-06,
-      "loss": 0.2749,
-      "step": 29
-    },
-    {
-      "epoch": 2.31,
-      "learning_rate": 2.780439060454756e-06,
-      "loss": 0.2664,
-      "step": 30
-    },
-    {
-      "epoch": 2.38,
-      "learning_rate": 2.2196424568156073e-06,
-      "loss": 0.2852,
-      "step": 31
-    },
-    {
-      "epoch": 2.46,
-      "learning_rate": 1.7149035075615795e-06,
-      "loss": 0.2876,
-      "step": 32
-    },
-    {
-      "epoch": 2.54,
-      "learning_rate": 1.2698588683881185e-06,
-      "loss": 0.2691,
-      "step": 33
-    },
-    {
-      "epoch": 2.62,
-      "learning_rate": 8.87715096118642e-07,
-      "loss": 0.2448,
-      "step": 34
-    },
-    {
-      "epoch": 2.69,
-      "learning_rate": 5.71225545389158e-07,
-      "loss": 0.2784,
-      "step": 35
-    },
-    {
-      "epoch": 2.77,
-      "learning_rate": 3.226705306650113e-07,
-      "loss": 0.2722,
-      "step": 36
-    },
-    {
-      "epoch": 2.85,
-      "learning_rate": 1.4384089652291544e-07,
-      "loss": 0.2862,
-      "step": 37
-    },
-    {
-      "epoch": 2.92,
-      "learning_rate": 3.602511457473479e-08,
-      "loss": 0.2631,
-      "step": 38
+      "learning_rate": 1e-05,
+      "loss": 1.0767,
+      "step": 2
     },
     {
       "epoch": 3.0,
       "learning_rate": 0.0,
-      "loss": 0.2251,
-      "step": 39
-    },
-    {
-      "epoch": 3.0,
-      "step": 39,
-      "total_flos": 2.4951731691454464e+16,
-      "train_loss": 0.3409684506746439,
-      "train_runtime": 1410.2975,
-      "train_samples_per_second": 3.54,
-      "train_steps_per_second": 0.028
+      "loss": 0.7536,
+      "step": 3
     },
     {
       "epoch": 3.0,
-      "eval_loss": 0.2879543900489807,
-      "eval_runtime": 10.2141,
-      "eval_samples_per_second": 40.728,
-      "eval_steps_per_second": 1.273,
-      "step": 39
+      "step": 3,
+      "total_flos": 2327628803473408.0,
+      "train_loss": 0.9601547519365946,
+      "train_runtime": 1446.9004,
+      "train_samples_per_second": 0.531,
+      "train_steps_per_second": 0.002
     }
   ],
-  "max_steps": 39,
+  "max_steps": 3,
   "num_train_epochs": 3,
-  "total_flos": 2.4951731691454464e+16,
+  "total_flos": 2327628803473408.0,
   "trial_name": null,
   "trial_params": null
 }

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:0307ecb82ecfd276a98da54f39915387c1581a2d03ada66ef5a4af4653463a47
-size 3771
+oid sha256:a37c01339dfc23b560f46c214c764dcc232eff864973e5037cb88d8c3da23be3
+size 3835