Upload tokenizer

README.md

@@ -1,5 +1,11 @@
 ---
+language:
+- en
+- es
+- fr
+- de
 library_name: tokenizers
+license: cc-by-4.0
 tags:
 - kl3m
 - kl3m-003
@@ -7,12 +13,6 @@ tags:
 - legal
 - financial
 date: '2024-03-15T00:00:00.000Z'
-license: cc-by-4.0
-language:
-- en
-- es
-- fr
-- de
 ---
 
 # kl3m-003-64k tokenizer

special_tokens_map.json

@@ -6,6 +6,13 @@
     "rstrip": false,
     "single_word": false
   },
+  "cls_token": {
+    "content": "<|cls|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "eos_token": {
     "content": "<|end|>",
     "lstrip": false,
@@ -27,6 +34,13 @@
     "rstrip": false,
     "single_word": false
   },
+  "sep_token": {
+    "content": "<|sep|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
   "unk_token": {
     "content": "<|unk|>",
     "lstrip": false,

tokenizer.json

@@ -1,14 +1,7 @@
 {
   "version": "1.0",
   "truncation": null,
-  "padding": {
-    "strategy": "BatchLongest",
-    "direction": "Left",
-    "pad_to_multiple_of": null,
-    "pad_id": 0,
-    "pad_type_id": 0,
-    "pad_token": "<|pad|>"
-  },
+  "padding": null,
   "added_tokens": [
     {
       "id": 0,

tokenizer_config.json

@@ -35709,15 +35709,24 @@
     }
   },
   "bos_token": "<|start|>",
+  "bos_token_id": 0,
   "clean_up_tokenization_spaces": false,
+  "cls_token": "<|cls|>",
+  "cls_token_id": 5,
   "eos_token": "<|end|>",
+  "eos_token_id": 1,
   "mask_token": "<|mask|>",
+  "mask_token_id": 6,
   "max_length": null,
-  "model_max_length": 1000000000000000019884624838656,
+  "model_max_length": 1048576,
   "pad_to_multiple_of": null,
   "pad_token": "<|pad|>",
+  "pad_token_id": 2,
   "pad_token_type_id": 0,
   "padding_side": "left",
-  "tokenizer_class": "GPTNeoXTokenizer",
-  "unk_token": "<|unk|>"
+  "sep_token": "<|sep|>",
+  "sep_token_id": 4,
+  "tokenizer_class": "PreTrainedTokenizerFast",
+  "unk_token": "<|unk|>",
+  "unk_token_id": 3
 }