Upload tokenizer

added_tokens.json

@@ -1506,6 +1506,7 @@
   "<|as|>": 50350,
   "<|az|>": 50304,
   "<|ba|>": 50355,
+  "<|begin_of_text|>": 51868,
   "<|be|>": 50330,
   "<|bg|>": 50292,
   "<|bn|>": 50302,
@@ -1518,12 +1519,14 @@
   "<|da|>": 50285,
   "<|de|>": 50261,
   "<|el|>": 50281,
+  "<|end_of_text|>": 51869,
   "<|endoftext|>": 50257,
   "<|en|>": 50259,
   "<|es|>": 50262,
   "<|et|>": 50307,
   "<|eu|>": 50310,
   "<|fa|>": 50300,
+  "<|finetune_right_pad_id|>": 51870,
   "<|fi|>": 50277,
   "<|fo|>": 50338,
   "<|fr|>": 50265,

tokenizer_config.json

@@ -12888,6 +12888,30 @@
       "rstrip": false,
       "single_word": false,
       "special": false
+    },
+    "51868": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51869": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
+    },
+    "51870": {
+      "content": "<|finetune_right_pad_id|>",
+      "lstrip": false,
+      "normalized": true,
+      "rstrip": false,
+      "single_word": false,
+      "special": false
     }
   },
   "additional_special_tokens": [