patrickvonplaten commited on
Commit
63750ab
1 Parent(s): 8237157

correct tok files

Browse files
Files changed (2) hide show
  1. config.json +2 -1
  2. tokenizer_config.json +1 -1
config.json CHANGED
@@ -35,5 +35,6 @@
35
  "use_cache": true,
36
  "vocab_size": 96103,
37
  "num_beams": 5,
38
- "length_penalty": 0.8
 
39
  }
 
35
  "use_cache": true,
36
  "vocab_size": 96103,
37
  "num_beams": 5,
38
+ "length_penalty": 0.8,
39
+ "tokenizer_class": "PegasusTokenizer"
40
  }
tokenizer_config.json CHANGED
@@ -1 +1 @@
1
- {"bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": {"content": "[MASK]", "single_word": false, "lstrip": true, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}}
 
1
+ {"pad_token": {"content": "<pad>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "eos_token": {"content": "</s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "unk_token": {"content": "<unk>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "mask_token": "[MASK]", "mask_token_sent": null, "offset": 0, "additional_special_tokens": [], "bos_token": {"content": "<s>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "sep_token": {"content": "[SEP]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "cls_token": {"content": "[CLS]", "single_word": false, "lstrip": false, "rstrip": false, "normalized": true, "__type": "AddedToken"}, "special_tokens_map_file": "/home/patrick/.cache/huggingface/transformers/b548e984b09823ed5cea0e622c0ec194a7e07b3c5d2e6b48a7cde84bd179cc7c.4eda581f816a0a941629106f0338c957910ce4839ecf7e3e743bb79523bf7249", "name_or_path": "./bigbird-pegasus-large-arxiv", "model_max_length": 4096}