jeffra commited on
Commit
5bc8dc3
1 Parent(s): c16aa90

Upload special_tokens_map.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. special_tokens_map.json +18 -6
special_tokens_map.json CHANGED
@@ -1,19 +1,31 @@
1
  {
2
  "additional_special_tokens": [
3
- "<|im_start|>",
4
- "<|im_end|>"
 
 
 
 
 
 
 
 
 
 
 
 
5
  ],
6
  "bos_token": {
7
- "content": "<s>",
8
  "lstrip": false,
9
- "normalized": true,
10
  "rstrip": false,
11
  "single_word": false
12
  },
13
  "eos_token": {
14
- "content": "</s>",
15
  "lstrip": false,
16
- "normalized": true,
17
  "rstrip": false,
18
  "single_word": false
19
  },
 
1
  {
2
  "additional_special_tokens": [
3
+ {
4
+ "content": "<s>",
5
+ "lstrip": false,
6
+ "normalized": false,
7
+ "rstrip": false,
8
+ "single_word": false
9
+ },
10
+ {
11
+ "content": "</s>",
12
+ "lstrip": false,
13
+ "normalized": false,
14
+ "rstrip": false,
15
+ "single_word": false
16
+ }
17
  ],
18
  "bos_token": {
19
+ "content": "<|im_start|>",
20
  "lstrip": false,
21
+ "normalized": false,
22
  "rstrip": false,
23
  "single_word": false
24
  },
25
  "eos_token": {
26
+ "content": "<|im_end|>",
27
  "lstrip": false,
28
+ "normalized": false,
29
  "rstrip": false,
30
  "single_word": false
31
  },