Upload folder using huggingface_hub

.ipynb_checkpoints/tokenizer-checkpoint.json

@@ -150,6 +150,12 @@
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
@@ -158,12 +164,24 @@
       }
     ],
     "pair": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 1
+        }
+      },
       {
         "Sequence": {
           "id": "B",
@@ -171,7 +189,17 @@
         }
       }
     ],
-    "special_tokens": {}
+    "special_tokens": {
+      "<s>": {
+        "id": "<s>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "<s>"
+        ]
+      }
+    }
   },
   "decoder": {
     "type": "Sequence",

.ipynb_checkpoints/tokenizer_config-checkpoint.json

@@ -117,7 +117,7 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{{- bos_token }}\n{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,

model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:51b5c641342f9faa9cd86fa5fda041708c9d16e17a0c698c73cb82c729335e69
+oid sha256:651ca57ba055c9ba9df28e566df37b77bd7745e9f456c40753fbf36a92378f8a
 size 45437480

tokenizer.json

@@ -150,6 +150,12 @@
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
@@ -158,12 +164,24 @@
       }
     ],
     "pair": [
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 0
+        }
+      },
       {
         "Sequence": {
           "id": "A",
           "type_id": 0
         }
       },
+      {
+        "SpecialToken": {
+          "id": "<s>",
+          "type_id": 1
+        }
+      },
       {
         "Sequence": {
           "id": "B",
@@ -171,7 +189,17 @@
         }
       }
     ],
-    "special_tokens": {}
+    "special_tokens": {
+      "<s>": {
+        "id": "<s>",
+        "ids": [
+          1
+        ],
+        "tokens": [
+          "<s>"
+        ]
+      }
+    }
   },
   "decoder": {
     "type": "Sequence",

tokenizer_config.json

@@ -117,7 +117,7 @@
     }
   },
   "bos_token": "<s>",
-  "chat_template": "{{- bos_token }}\n{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
+  "chat_template": "{% for message in messages %}{% if message['role'] == 'system' %}{{'<|system|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\n' + message['content'] + '<|end|>\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n' + message['content'] + '<|end|>\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n' }}{% else %}{{ eos_token }}{% endif %}",
   "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "legacy": false,