lysandre HF staff commited on
Commit
029c042
1 Parent(s): b4f0c87

Create tokenizer_config.json

Browse files
Files changed (1) hide show
  1. tokenizer_config.json +1 -0
tokenizer_config.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"do_lowercase_and_remove_accent": true, "id2lang": {"0": "ar", "1": "bg", "2": "de", "3": "el", "4": "en", "5": "es", "6": "fr", "7": "hi", "8": "ru", "9": "sw", "10": "th", "11": "tr", "12": "ur", "13": "vi", "14": "zh"}, "lang2id": {"ar": 0, "bg": 1, "de": 2, "el": 3, "en": 4, "es": 5, "fr": 6, "hi": 7, "ru": 8, "sw": 9, "th": 10, "tr": 11, "ur": 12, "vi": 13, "zh": 14}, "model_max_length": 512}