intervitens commited on
Commit
76e349d
1 Parent(s): d29a005

Update tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +2 -2
tokenizer_config.json CHANGED
@@ -64,7 +64,7 @@
64
  "normalized": false,
65
  "rstrip": false,
66
  "single_word": false,
67
- "special": false
68
  },
69
  "8": {
70
  "content": "<|im_end|>",
@@ -72,7 +72,7 @@
72
  "normalized": false,
73
  "rstrip": false,
74
  "single_word": false,
75
- "special": false
76
  },
77
  "9": {
78
  "content": "<unused2>",
 
64
  "normalized": false,
65
  "rstrip": false,
66
  "single_word": false,
67
+ "special": true
68
  },
69
  "8": {
70
  "content": "<|im_end|>",
 
72
  "normalized": false,
73
  "rstrip": false,
74
  "single_word": false,
75
+ "special": true
76
  },
77
  "9": {
78
  "content": "<unused2>",