Upload tokenizer

added_tokens.json

@@ -537,5 +537,7 @@
   "\\xlongequal": 32634,
   "\\xrightarrow": 32635,
   "\\your": 32636,
-  "\\zeta": 32637
+  "\\zeta": 32637,
+  "{": 32639,
+  "}": 32640
 }

tokenizer.json

@@ -5794,6 +5794,24 @@
       "rstrip": false,
       "normalized": true,
       "special": false
+    },
+    {
+      "id": 32639,
+      "content": "{",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
+    },
+    {
+      "id": 32640,
+      "content": "}",
+      "single_word": false,
+      "lstrip": false,
+      "rstrip": false,
+      "normalized": true,
+      "special": false
     }
   ],
   "normalizer": {

tokenizer_config.json

@@ -105,7 +105,7 @@
   "eos_token": "</s>",
   "extra_ids": 100,
   "model_max_length": 512,
-  "name_or_path": "./models/definition_naming_model_2024_06_03_2",
+  "name_or_path": "./models/definition_naming_model_2024_06_07",
   "pad_token": "<pad>",
   "special_tokens_map_file": null,
   "tokenizer_class": "T5Tokenizer",