add tokenizer

.gitignore

@@ -0,0 +1 @@
+checkpoint-*/

runs/Feb13_18-05-01_9bf831cc8184/1644775571.0542042/events.out.tfevents.1644775571.9bf831cc8184.80.1

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:21c625ac805feaaacbaad471055e6a3d88f07c19ccf56dffbe33e659ac7d5c10
+size 4555

runs/Feb13_18-05-01_9bf831cc8184/events.out.tfevents.1644775571.9bf831cc8184.80.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c676be5d6c533ea0932647ad0ca60f954fa9d44c97686cf50167a291bde3f4be
+size 4464

runs/Feb13_18-07-24_9bf831cc8184/1644775702.5796926/events.out.tfevents.1644775702.9bf831cc8184.80.3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ef7519b6d9b33a735b0282c7259748671cf481df73d3784cfe691b8eeaf902b
+size 4555

runs/Feb13_18-07-24_9bf831cc8184/events.out.tfevents.1644775702.9bf831cc8184.80.2

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3844468cc66bf6e2ebe7becf92676f4e85ea90fdc935e2962ebe8fdda3c2ff69
+size 4464

runs/Feb13_18-08-42_9bf831cc8184/1644775776.689278/events.out.tfevents.1644775776.9bf831cc8184.80.5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2eee2a3c9bc7e78dcbeecea0ba8d63ee8a12c1adb79e08ffdd06730ce9e1a0fb
+size 4555

runs/Feb13_18-08-42_9bf831cc8184/events.out.tfevents.1644775776.9bf831cc8184.80.4

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d38375050ae4d7f1063605b0dfa981f7000a5ce6ebdd480bf321ce09e4ca89f
+size 4464

runs/Feb13_18-10-00_9bf831cc8184/1644775856.1088345/events.out.tfevents.1644775856.9bf831cc8184.80.7

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:402b0e388daf554474671d3c7090ef6230957f14331e037166f35cba43d93387
+size 4555

runs/Feb13_18-10-00_9bf831cc8184/events.out.tfevents.1644775856.9bf831cc8184.80.6

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbf6cb751fe5ab0f050b6d3539276042ae6c9b733c4274506d16fa2a0091c48c
+size 4464

runs/Feb13_18-11-16_9bf831cc8184/1644775930.0677655/events.out.tfevents.1644775930.9bf831cc8184.80.9

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f5c48821029e02669779e35e3ab9606bcfab10b3e998951dc00355ea48fbf2d
+size 4555

runs/Feb13_18-11-16_9bf831cc8184/events.out.tfevents.1644775930.9bf831cc8184.80.8

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e972113f335dcb806e2e29e4bf72fde0866e049317ae6bd0d88585801227db99
+size 4465

runs/Feb13_18-12-46_9bf831cc8184/1644776016.462528/events.out.tfevents.1644776016.9bf831cc8184.80.11

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c10d4671559c45bdb7bef54483b85e249265549f366d40fa811825a3c6195e37
+size 4555

runs/Feb13_18-12-46_9bf831cc8184/1644776654.586405/events.out.tfevents.1644776654.9bf831cc8184.80.12

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:853bed69d608a21526aa9781bdb639e444a1d536ae48c36da6b30b0813f81182
+size 4555

runs/Feb13_18-12-46_9bf831cc8184/1644776956.7464147/events.out.tfevents.1644776956.9bf831cc8184.80.13

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:368fbfa50eedfe250de0be47cad5707c6be8d0d8ed29c2758dbbac5ee2a48568
+size 4555

runs/Feb13_18-12-46_9bf831cc8184/1644777198.6019511/events.out.tfevents.1644777198.9bf831cc8184.80.14

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5b8865ddbfdb083a27aca0be2a9ab2cfeadb7822d7f9910107359aeedeb78c9f
+size 4555

runs/Feb13_18-12-46_9bf831cc8184/events.out.tfevents.1644776016.9bf831cc8184.80.10

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e77306d7fc2e7631222ad534af44dc8f03fec8493a8680ff7be2e36ce7956238
+size 17740

runs/Feb13_18-34-08_9bf831cc8184/1644777304.1245875/events.out.tfevents.1644777304.9bf831cc8184.80.16

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d60718e26d9fcd562dc5f78b5da710364245cf9225e9cfd39ba2170b736b413e
+size 4555

runs/Feb13_18-34-08_9bf831cc8184/events.out.tfevents.1644777304.9bf831cc8184.80.15

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aa644aa110dc45a5d02325f1f37b84f173a043b4c120914f6215a764845145b4
+size 4465

runs/Feb13_18-35-22_9bf831cc8184/1644777373.3517425/events.out.tfevents.1644777373.9bf831cc8184.80.18

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db18c6e6347de6f39489755a33ca6b7eb61527d8d0c8ae80c9470298b2d8c2f1
+size 4555

runs/Feb13_18-35-22_9bf831cc8184/events.out.tfevents.1644777373.9bf831cc8184.80.17

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4bd1cdc431dc1bd9abdad489743b3b85bf3c6cb1bc3b180f36795bae26c2c8f3
+size 4464

runs/Feb13_18-36-34_9bf831cc8184/1644777445.9582272/events.out.tfevents.1644777445.9bf831cc8184.80.20

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4e344ccf8ac327fda0d7d37aeab6bfd4c58d417740829da9137a39d55e4e1d21
+size 4555

runs/Feb13_18-36-34_9bf831cc8184/1644778072.6220896/events.out.tfevents.1644778072.9bf831cc8184.80.21

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39ccbb3609821d6f9952fabac8cbf4e97ab356bc2fa2995cb5cfc197545cd65a
+size 4555

runs/Feb13_18-36-34_9bf831cc8184/1644778166.3728943/events.out.tfevents.1644778166.9bf831cc8184.80.22

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:eb192643bc92fde7ab4e9402104357482a2dcf60e72a2b3837499e836c2f8bb6
+size 4555

runs/Feb13_18-36-34_9bf831cc8184/events.out.tfevents.1644777445.9bf831cc8184.80.19

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:754af687b93a61ef146b379f3758dfbf62af7be2c4d41479d449113ee0c9e88b
+size 13315

runs/Feb13_18-49-48_9bf831cc8184/1644778252.0627105/events.out.tfevents.1644778252.9bf831cc8184.80.24

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14962f457a915d807f8f88affd533e935f672b0fc7c85a1d210d211ed1a194c9
+size 4555

runs/Feb13_18-49-48_9bf831cc8184/events.out.tfevents.1644778252.9bf831cc8184.80.23

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b1e78eedde6838c97e4a5855e7034bfc96a702dbd318f0f47d63994dfba69e7
+size 4465

runs/Feb13_18-51-28_9bf831cc8184/1644778348.02025/events.out.tfevents.1644778348.9bf831cc8184.80.26

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c9146f9673a4bb4faa7c680b76e1dbf2e197d81e8774d27bb98b1f2080e35705
+size 4555

runs/Feb13_18-51-28_9bf831cc8184/events.out.tfevents.1644778348.9bf831cc8184.80.25

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79430c19ac8664c35f55a33a2fcc60f02ab3e53cc787b959cd418d939dab7eaf
+size 4465

runs/Feb13_18-56-21_9bf831cc8184/1644778638.6797016/events.out.tfevents.1644778638.9bf831cc8184.80.28

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0d9a48500bfda4e1231a8337b6b45a723db9d608452bf3b33c04e62a7bb9856
+size 4555

runs/Feb13_18-56-21_9bf831cc8184/events.out.tfevents.1644778638.9bf831cc8184.80.27

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a83eef15313da1b7ee3a0915658204c627c7f41593774130500a8c35ecb331c1
+size 4465

runs/Feb13_18-57-35_9bf831cc8184/1644778715.355512/events.out.tfevents.1644778715.9bf831cc8184.80.30

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:abeda431962bbff2b435c55e1665e56b1a7b5f89658b0013a4789f9babbf0af5
+size 4555

runs/Feb13_18-57-35_9bf831cc8184/1644779043.8606036/events.out.tfevents.1644779043.9bf831cc8184.80.31

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fe9aafc9a0bccaa7a786e536e0f0c8be31377c04e7a204f5352e6f9d94bb0adf
+size 4555

runs/Feb13_18-57-35_9bf831cc8184/events.out.tfevents.1644778715.9bf831cc8184.80.29

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b149e3fbf6ef0625f5669b1b3b3a5044a8740b502a0390d3dd5dee6fdb9d8996
+size 8888

vocab.json

@@ -1 +1 @@
-{"q": 0, "x": 1, "ƙ": 2, "“": 3, "y": 4, "f": 5, "ɓ": 6, "’": 7, "h": 8, "d": 9, "t": 10, "u": 11, "n": 12, "r": 13, "a": 14, "j": 15, "ɗ": 16, "p": 18, "”": 19, "g": 20, "m": 21, "v": 22, "w": 23, "l": 24, "í": 25, "k": 26, "s": 27, "z": 28, "ʻ": 29, "i": 30, "o": 31, "'": 32, "—": 33, "ƴ": 34, "b": 35, "e": 36, "c": 37, "|": 17, "[UNK]": 38, "[PAD]": 39}
+{"m": 0, "r": 1, "w": 2, "”": 3, "a": 4, "l": 5, "u": 6, "“": 7, "x": 8, "ʻ": 9, "t": 10, "k": 11, "ƙ": 12, "ɗ": 13, "i": 14, "’": 15, "e": 16, "o": 17, "í": 18, "j": 19, "'": 20, "h": 21, "z": 22, "p": 23, "d": 24, "n": 25, "y": 26, "b": 27, "g": 28, "—": 29, "ƴ": 30, "s": 31, "f": 32, "q": 34, "v": 35, "ɓ": 36, "c": 37, "|": 33, "[UNK]": 38, "[PAD]": 39}