monologg commited on
Commit
67dd433
1 Parent(s): cb6e01e

fix: only use token_type id 0 for fast tokenizer

Browse files
Files changed (1) hide show
  1. tokenizer.json +2 -2
tokenizer.json CHANGED
@@ -68,8 +68,8 @@
68
  { "SpecialToken": { "id": "[CLS]", "type_id": 0 } },
69
  { "Sequence": { "id": "A", "type_id": 0 } },
70
  { "SpecialToken": { "id": "[SEP]", "type_id": 0 } },
71
- { "Sequence": { "id": "B", "type_id": 1 } },
72
- { "SpecialToken": { "id": "[SEP]", "type_id": 1 } }
73
  ],
74
  "special_tokens": {
75
  "[CLS]": { "id": "[CLS]", "ids": [0], "tokens": ["[CLS]"] },
 
68
  { "SpecialToken": { "id": "[CLS]", "type_id": 0 } },
69
  { "Sequence": { "id": "A", "type_id": 0 } },
70
  { "SpecialToken": { "id": "[SEP]", "type_id": 0 } },
71
+ { "Sequence": { "id": "B", "type_id": 0 } },
72
+ { "SpecialToken": { "id": "[SEP]", "type_id": 0 } }
73
  ],
74
  "special_tokens": {
75
  "[CLS]": { "id": "[CLS]", "ids": [0], "tokens": ["[CLS]"] },