Upload tokenizer

README.md

@@ -1,5 +1,6 @@
 ---
-license: cc-by-sa-4.0
+base_model:
+- openai-community/gpt2
 datasets:
 - speechcolab/gigaspeech
 - parler-tts/mls_eng_10k
@@ -11,10 +12,9 @@ datasets:
 language:
 - en
 - hi
-base_model:
-- openai-community/gpt2
-pipeline_tag: text-to-speech
 library_name: transformers
+license: cc-by-sa-4.0
+pipeline_tag: text-to-speech
 ---
 
 | Platform | Link |

added_tokens.json

@@ -16386,85 +16386,85 @@
   "[continue]": 66644,
   "[convert]": 66643,
   "[mimi]": 66642,
-  "[spkr_expresso_ex01]": 66698,
-  "[spkr_expresso_ex02]": 66695,
-  "[spkr_expresso_ex03]": 66696,
-  "[spkr_expresso_ex04]": 66697,
-  "[spkr_hifi_tts_6097]": 66693,
-  "[spkr_hifi_tts_9017]": 66692,
-  "[spkr_hifi_tts_92]": 66694,
-  "[spkr_jenny_jenny]": 66699,
-  "[spkr_mls_eng_10k_10191]": 66657,
-  "[spkr_mls_eng_10k_10244]": 66676,
-  "[spkr_mls_eng_10k_10463]": 66691,
-  "[spkr_mls_eng_10k_107]": 66664,
-  "[spkr_mls_eng_10k_10801]": 66648,
-  "[spkr_mls_eng_10k_11667]": 66671,
-  "[spkr_mls_eng_10k_11697]": 66658,
-  "[spkr_mls_eng_10k_1259]": 66665,
-  "[spkr_mls_eng_10k_1401]": 66659,
-  "[spkr_mls_eng_10k_1724]": 66669,
-  "[spkr_mls_eng_10k_204]": 66662,
-  "[spkr_mls_eng_10k_2156]": 66651,
-  "[spkr_mls_eng_10k_2234]": 66675,
-  "[spkr_mls_eng_10k_2607]": 66668,
-  "[spkr_mls_eng_10k_3003]": 66679,
-  "[spkr_mls_eng_10k_3157]": 66654,
-  "[spkr_mls_eng_10k_3488]": 66656,
-  "[spkr_mls_eng_10k_3595]": 66686,
-  "[spkr_mls_eng_10k_3645]": 66652,
-  "[spkr_mls_eng_10k_3657]": 66688,
-  "[spkr_mls_eng_10k_3681]": 66672,
-  "[spkr_mls_eng_10k_3699]": 66678,
-  "[spkr_mls_eng_10k_3967]": 66689,
-  "[spkr_mls_eng_10k_4018]": 66660,
-  "[spkr_mls_eng_10k_4064]": 66687,
-  "[spkr_mls_eng_10k_4078]": 66650,
-  "[spkr_mls_eng_10k_4174]": 66680,
-  "[spkr_mls_eng_10k_4788]": 66670,
-  "[spkr_mls_eng_10k_5304]": 66683,
-  "[spkr_mls_eng_10k_5717]": 66655,
-  "[spkr_mls_eng_10k_5968]": 66667,
-  "[spkr_mls_eng_10k_5985]": 66677,
-  "[spkr_mls_eng_10k_6454]": 66647,
-  "[spkr_mls_eng_10k_6497]": 66685,
-  "[spkr_mls_eng_10k_6660]": 66681,
-  "[spkr_mls_eng_10k_6670]": 66674,
-  "[spkr_mls_eng_10k_6671]": 66673,
-  "[spkr_mls_eng_10k_6687]": 66684,
-  "[spkr_mls_eng_10k_7756]": 66649,
-  "[spkr_mls_eng_10k_7879]": 66661,
-  "[spkr_mls_eng_10k_8011]": 66690,
-  "[spkr_mls_eng_10k_8138]": 66666,
-  "[spkr_mls_eng_10k_8677]": 66663,
-  "[spkr_mls_eng_10k_8713]": 66653,
-  "[spkr_mls_eng_10k_9017]": 66682,
-  "[spkr_shrutilipi_hi_shrutilipi]": 66720,
+  "[spkr_0]": 66647,
+  "[spkr_10]": 66657,
+  "[spkr_11]": 66658,
+  "[spkr_12]": 66659,
+  "[spkr_13]": 66660,
+  "[spkr_14]": 66661,
+  "[spkr_15]": 66662,
+  "[spkr_16]": 66663,
+  "[spkr_17]": 66664,
+  "[spkr_18]": 66665,
+  "[spkr_19]": 66666,
+  "[spkr_1]": 66648,
+  "[spkr_20]": 66667,
+  "[spkr_21]": 66668,
+  "[spkr_22]": 66669,
+  "[spkr_23]": 66670,
+  "[spkr_24]": 66671,
+  "[spkr_25]": 66672,
+  "[spkr_26]": 66673,
+  "[spkr_27]": 66674,
+  "[spkr_28]": 66675,
+  "[spkr_29]": 66676,
+  "[spkr_2]": 66649,
+  "[spkr_30]": 66677,
+  "[spkr_31]": 66678,
+  "[spkr_32]": 66679,
+  "[spkr_33]": 66680,
+  "[spkr_34]": 66681,
+  "[spkr_35]": 66682,
+  "[spkr_36]": 66683,
+  "[spkr_37]": 66684,
+  "[spkr_38]": 66685,
+  "[spkr_39]": 66686,
+  "[spkr_3]": 66650,
+  "[spkr_40]": 66687,
+  "[spkr_41]": 66688,
+  "[spkr_42]": 66689,
+  "[spkr_43]": 66690,
+  "[spkr_44]": 66691,
+  "[spkr_45]": 66692,
+  "[spkr_46]": 66693,
+  "[spkr_47]": 66694,
+  "[spkr_48]": 66695,
+  "[spkr_49]": 66696,
+  "[spkr_4]": 66651,
+  "[spkr_50]": 66697,
+  "[spkr_51]": 66698,
+  "[spkr_52]": 66699,
+  "[spkr_53]": 66700,
+  "[spkr_54]": 66701,
+  "[spkr_55]": 66702,
+  "[spkr_56]": 66703,
+  "[spkr_57]": 66704,
+  "[spkr_58]": 66705,
+  "[spkr_59]": 66706,
+  "[spkr_5]": 66652,
+  "[spkr_60]": 66707,
+  "[spkr_61]": 66708,
+  "[spkr_62]": 66709,
+  "[spkr_63]": 66710,
+  "[spkr_64]": 66711,
+  "[spkr_65]": 66712,
+  "[spkr_66]": 66713,
+  "[spkr_67]": 66714,
+  "[spkr_68]": 66715,
+  "[spkr_69]": 66716,
+  "[spkr_6]": 66653,
+  "[spkr_70]": 66717,
+  "[spkr_71]": 66718,
+  "[spkr_72]": 66719,
+  "[spkr_73]": 66720,
+  "[spkr_74]": 66721,
+  "[spkr_75]": 66722,
+  "[spkr_76]": 66723,
+  "[spkr_77]": 66724,
+  "[spkr_7]": 66654,
+  "[spkr_8]": 66655,
+  "[spkr_9]": 66656,
   "[spkr_unk]": 66646,
-  "[spkr_youtube_webds_en_akshat]": 66705,
-  "[spkr_youtube_webds_en_attenborough]": 66723,
-  "[spkr_youtube_webds_en_derekperkins]": 66721,
-  "[spkr_youtube_webds_en_historyofindia]": 66710,
-  "[spkr_youtube_webds_en_mkbhd]": 66714,
-  "[spkr_youtube_webds_en_mukesh]": 66722,
-  "[spkr_youtube_webds_en_secondhandstories]": 66715,
-  "[spkr_youtube_webds_en_storiesofmahabharatha]": 66716,
-  "[spkr_youtube_webds_en_teded]": 66703,
-  "[spkr_youtube_webds_hi_a2motivation]": 66717,
-  "[spkr_youtube_webds_hi_akshat]": 66719,
-  "[spkr_youtube_webds_hi_dhruvrathee]": 66706,
-  "[spkr_youtube_webds_hi_hindiaudiobooks]": 66709,
-  "[spkr_youtube_webds_hi_kabitaskitchen]": 66700,
-  "[spkr_youtube_webds_hi_mrbeast]": 66718,
-  "[spkr_youtube_webds_hi_neelimaaudiobooks]": 66707,
-  "[spkr_youtube_webds_hi_physicswallah]": 66704,
-  "[spkr_youtube_webds_hi_pmmodi]": 66713,
-  "[spkr_youtube_webds_hi_ranveerallahbadia]": 66708,
-  "[spkr_youtube_webds_hi_sandeepmaheshwari]": 66712,
-  "[spkr_youtube_webds_hi_technicalguruji]": 66702,
-  "[spkr_youtube_webds_hi_unacademyjee]": 66701,
-  "[spkr_youtube_webds_hi_vivekbindra]": 66711,
-  "[spkr_youtube_webds_hi_warikoo]": 66724,
   "[stop]": 66645,
   "[text]": 66641
 }

special_tokens_map.json

@@ -1,23 +1,5 @@
 {
-  "bos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "eos_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  },
-  "unk_token": {
-    "content": "<|endoftext|>",
-    "lstrip": false,
-    "normalized": true,
-    "rstrip": false,
-    "single_word": false
-  }
+  "bos_token": "<|endoftext|>",
+  "eos_token": "<|endoftext|>",
+  "unk_token": "<|endoftext|>"
 }

tokenizer.json

@@ -10,7 +10,7 @@
       "lstrip": false,
       "rstrip": false,
       "normalized": true,
-      "special": true
+      "special": false
     },
     {
       "id": 50257,
@@ -147524,7 +147524,7 @@
     },
     {
       "id": 66647,
-      "content": "[spkr_mls_eng_10k_6454]",
+      "content": "[spkr_0]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147533,7 +147533,7 @@
     },
     {
       "id": 66648,
-      "content": "[spkr_mls_eng_10k_10801]",
+      "content": "[spkr_1]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147542,7 +147542,7 @@
     },
     {
       "id": 66649,
-      "content": "[spkr_mls_eng_10k_7756]",
+      "content": "[spkr_2]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147551,7 +147551,7 @@
     },
     {
       "id": 66650,
-      "content": "[spkr_mls_eng_10k_4078]",
+      "content": "[spkr_3]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147560,7 +147560,7 @@
     },
     {
       "id": 66651,
-      "content": "[spkr_mls_eng_10k_2156]",
+      "content": "[spkr_4]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147569,7 +147569,7 @@
     },
     {
       "id": 66652,
-      "content": "[spkr_mls_eng_10k_3645]",
+      "content": "[spkr_5]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147578,7 +147578,7 @@
     },
     {
       "id": 66653,
-      "content": "[spkr_mls_eng_10k_8713]",
+      "content": "[spkr_6]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147587,7 +147587,7 @@
     },
     {
       "id": 66654,
-      "content": "[spkr_mls_eng_10k_3157]",
+      "content": "[spkr_7]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147596,7 +147596,7 @@
     },
     {
       "id": 66655,
-      "content": "[spkr_mls_eng_10k_5717]",
+      "content": "[spkr_8]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147605,7 +147605,7 @@
     },
     {
       "id": 66656,
-      "content": "[spkr_mls_eng_10k_3488]",
+      "content": "[spkr_9]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147614,7 +147614,7 @@
     },
     {
       "id": 66657,
-      "content": "[spkr_mls_eng_10k_10191]",
+      "content": "[spkr_10]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147623,7 +147623,7 @@
     },
     {
       "id": 66658,
-      "content": "[spkr_mls_eng_10k_11697]",
+      "content": "[spkr_11]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147632,7 +147632,7 @@
     },
     {
       "id": 66659,
-      "content": "[spkr_mls_eng_10k_1401]",
+      "content": "[spkr_12]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147641,7 +147641,7 @@
     },
     {
       "id": 66660,
-      "content": "[spkr_mls_eng_10k_4018]",
+      "content": "[spkr_13]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147650,7 +147650,7 @@
     },
     {
       "id": 66661,
-      "content": "[spkr_mls_eng_10k_7879]",
+      "content": "[spkr_14]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147659,7 +147659,7 @@
     },
     {
       "id": 66662,
-      "content": "[spkr_mls_eng_10k_204]",
+      "content": "[spkr_15]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147668,7 +147668,7 @@
     },
     {
       "id": 66663,
-      "content": "[spkr_mls_eng_10k_8677]",
+      "content": "[spkr_16]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147677,7 +147677,7 @@
     },
     {
       "id": 66664,
-      "content": "[spkr_mls_eng_10k_107]",
+      "content": "[spkr_17]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147686,7 +147686,7 @@
     },
     {
       "id": 66665,
-      "content": "[spkr_mls_eng_10k_1259]",
+      "content": "[spkr_18]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147695,7 +147695,7 @@
     },
     {
       "id": 66666,
-      "content": "[spkr_mls_eng_10k_8138]",
+      "content": "[spkr_19]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147704,7 +147704,7 @@
     },
     {
       "id": 66667,
-      "content": "[spkr_mls_eng_10k_5968]",
+      "content": "[spkr_20]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147713,7 +147713,7 @@
     },
     {
       "id": 66668,
-      "content": "[spkr_mls_eng_10k_2607]",
+      "content": "[spkr_21]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147722,7 +147722,7 @@
     },
     {
       "id": 66669,
-      "content": "[spkr_mls_eng_10k_1724]",
+      "content": "[spkr_22]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147731,7 +147731,7 @@
     },
     {
       "id": 66670,
-      "content": "[spkr_mls_eng_10k_4788]",
+      "content": "[spkr_23]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147740,7 +147740,7 @@
     },
     {
       "id": 66671,
-      "content": "[spkr_mls_eng_10k_11667]",
+      "content": "[spkr_24]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147749,7 +147749,7 @@
     },
     {
       "id": 66672,
-      "content": "[spkr_mls_eng_10k_3681]",
+      "content": "[spkr_25]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147758,7 +147758,7 @@
     },
     {
       "id": 66673,
-      "content": "[spkr_mls_eng_10k_6671]",
+      "content": "[spkr_26]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147767,7 +147767,7 @@
     },
     {
       "id": 66674,
-      "content": "[spkr_mls_eng_10k_6670]",
+      "content": "[spkr_27]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147776,7 +147776,7 @@
     },
     {
       "id": 66675,
-      "content": "[spkr_mls_eng_10k_2234]",
+      "content": "[spkr_28]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147785,7 +147785,7 @@
     },
     {
       "id": 66676,
-      "content": "[spkr_mls_eng_10k_10244]",
+      "content": "[spkr_29]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147794,7 +147794,7 @@
     },
     {
       "id": 66677,
-      "content": "[spkr_mls_eng_10k_5985]",
+      "content": "[spkr_30]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147803,7 +147803,7 @@
     },
     {
       "id": 66678,
-      "content": "[spkr_mls_eng_10k_3699]",
+      "content": "[spkr_31]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147812,7 +147812,7 @@
     },
     {
       "id": 66679,
-      "content": "[spkr_mls_eng_10k_3003]",
+      "content": "[spkr_32]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147821,7 +147821,7 @@
     },
     {
       "id": 66680,
-      "content": "[spkr_mls_eng_10k_4174]",
+      "content": "[spkr_33]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147830,7 +147830,7 @@
     },
     {
       "id": 66681,
-      "content": "[spkr_mls_eng_10k_6660]",
+      "content": "[spkr_34]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147839,7 +147839,7 @@
     },
     {
       "id": 66682,
-      "content": "[spkr_mls_eng_10k_9017]",
+      "content": "[spkr_35]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147848,7 +147848,7 @@
     },
     {
       "id": 66683,
-      "content": "[spkr_mls_eng_10k_5304]",
+      "content": "[spkr_36]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147857,7 +147857,7 @@
     },
     {
       "id": 66684,
-      "content": "[spkr_mls_eng_10k_6687]",
+      "content": "[spkr_37]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147866,7 +147866,7 @@
     },
     {
       "id": 66685,
-      "content": "[spkr_mls_eng_10k_6497]",
+      "content": "[spkr_38]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147875,7 +147875,7 @@
     },
     {
       "id": 66686,
-      "content": "[spkr_mls_eng_10k_3595]",
+      "content": "[spkr_39]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147884,7 +147884,7 @@
     },
     {
       "id": 66687,
-      "content": "[spkr_mls_eng_10k_4064]",
+      "content": "[spkr_40]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147893,7 +147893,7 @@
     },
     {
       "id": 66688,
-      "content": "[spkr_mls_eng_10k_3657]",
+      "content": "[spkr_41]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147902,7 +147902,7 @@
     },
     {
       "id": 66689,
-      "content": "[spkr_mls_eng_10k_3967]",
+      "content": "[spkr_42]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147911,7 +147911,7 @@
     },
     {
       "id": 66690,
-      "content": "[spkr_mls_eng_10k_8011]",
+      "content": "[spkr_43]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147920,7 +147920,7 @@
     },
     {
       "id": 66691,
-      "content": "[spkr_mls_eng_10k_10463]",
+      "content": "[spkr_44]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147929,7 +147929,7 @@
     },
     {
       "id": 66692,
-      "content": "[spkr_hifi_tts_9017]",
+      "content": "[spkr_45]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147938,7 +147938,7 @@
     },
     {
       "id": 66693,
-      "content": "[spkr_hifi_tts_6097]",
+      "content": "[spkr_46]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147947,7 +147947,7 @@
     },
     {
       "id": 66694,
-      "content": "[spkr_hifi_tts_92]",
+      "content": "[spkr_47]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147956,7 +147956,7 @@
     },
     {
       "id": 66695,
-      "content": "[spkr_expresso_ex02]",
+      "content": "[spkr_48]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147965,7 +147965,7 @@
     },
     {
       "id": 66696,
-      "content": "[spkr_expresso_ex03]",
+      "content": "[spkr_49]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147974,7 +147974,7 @@
     },
     {
       "id": 66697,
-      "content": "[spkr_expresso_ex04]",
+      "content": "[spkr_50]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147983,7 +147983,7 @@
     },
     {
       "id": 66698,
-      "content": "[spkr_expresso_ex01]",
+      "content": "[spkr_51]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -147992,7 +147992,7 @@
     },
     {
       "id": 66699,
-      "content": "[spkr_jenny_jenny]",
+      "content": "[spkr_52]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148001,7 +148001,7 @@
     },
     {
       "id": 66700,
-      "content": "[spkr_youtube_webds_hi_kabitaskitchen]",
+      "content": "[spkr_53]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148010,7 +148010,7 @@
     },
     {
       "id": 66701,
-      "content": "[spkr_youtube_webds_hi_unacademyjee]",
+      "content": "[spkr_54]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148019,7 +148019,7 @@
     },
     {
       "id": 66702,
-      "content": "[spkr_youtube_webds_hi_technicalguruji]",
+      "content": "[spkr_55]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148028,7 +148028,7 @@
     },
     {
       "id": 66703,
-      "content": "[spkr_youtube_webds_en_teded]",
+      "content": "[spkr_56]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148037,7 +148037,7 @@
     },
     {
       "id": 66704,
-      "content": "[spkr_youtube_webds_hi_physicswallah]",
+      "content": "[spkr_57]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148046,7 +148046,7 @@
     },
     {
       "id": 66705,
-      "content": "[spkr_youtube_webds_en_akshat]",
+      "content": "[spkr_58]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148055,7 +148055,7 @@
     },
     {
       "id": 66706,
-      "content": "[spkr_youtube_webds_hi_dhruvrathee]",
+      "content": "[spkr_59]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148064,7 +148064,7 @@
     },
     {
       "id": 66707,
-      "content": "[spkr_youtube_webds_hi_neelimaaudiobooks]",
+      "content": "[spkr_60]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148073,7 +148073,7 @@
     },
     {
       "id": 66708,
-      "content": "[spkr_youtube_webds_hi_ranveerallahbadia]",
+      "content": "[spkr_61]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148082,7 +148082,7 @@
     },
     {
       "id": 66709,
-      "content": "[spkr_youtube_webds_hi_hindiaudiobooks]",
+      "content": "[spkr_62]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148091,7 +148091,7 @@
     },
     {
       "id": 66710,
-      "content": "[spkr_youtube_webds_en_historyofindia]",
+      "content": "[spkr_63]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148100,7 +148100,7 @@
     },
     {
       "id": 66711,
-      "content": "[spkr_youtube_webds_hi_vivekbindra]",
+      "content": "[spkr_64]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148109,7 +148109,7 @@
     },
     {
       "id": 66712,
-      "content": "[spkr_youtube_webds_hi_sandeepmaheshwari]",
+      "content": "[spkr_65]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148118,7 +148118,7 @@
     },
     {
       "id": 66713,
-      "content": "[spkr_youtube_webds_hi_pmmodi]",
+      "content": "[spkr_66]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148127,7 +148127,7 @@
     },
     {
       "id": 66714,
-      "content": "[spkr_youtube_webds_en_mkbhd]",
+      "content": "[spkr_67]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148136,7 +148136,7 @@
     },
     {
       "id": 66715,
-      "content": "[spkr_youtube_webds_en_secondhandstories]",
+      "content": "[spkr_68]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148145,7 +148145,7 @@
     },
     {
       "id": 66716,
-      "content": "[spkr_youtube_webds_en_storiesofmahabharatha]",
+      "content": "[spkr_69]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148154,7 +148154,7 @@
     },
     {
       "id": 66717,
-      "content": "[spkr_youtube_webds_hi_a2motivation]",
+      "content": "[spkr_70]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148163,7 +148163,7 @@
     },
     {
       "id": 66718,
-      "content": "[spkr_youtube_webds_hi_mrbeast]",
+      "content": "[spkr_71]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148172,7 +148172,7 @@
     },
     {
       "id": 66719,
-      "content": "[spkr_youtube_webds_hi_akshat]",
+      "content": "[spkr_72]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148181,7 +148181,7 @@
     },
     {
       "id": 66720,
-      "content": "[spkr_shrutilipi_hi_shrutilipi]",
+      "content": "[spkr_73]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148190,7 +148190,7 @@
     },
     {
       "id": 66721,
-      "content": "[spkr_youtube_webds_en_derekperkins]",
+      "content": "[spkr_74]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148199,7 +148199,7 @@
     },
     {
       "id": 66722,
-      "content": "[spkr_youtube_webds_en_mukesh]",
+      "content": "[spkr_75]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148208,7 +148208,7 @@
     },
     {
       "id": 66723,
-      "content": "[spkr_youtube_webds_en_attenborough]",
+      "content": "[spkr_76]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,
@@ -148217,7 +148217,7 @@
     },
     {
       "id": 66724,
-      "content": "[spkr_youtube_webds_hi_warikoo]",
+      "content": "[spkr_77]",
       "single_word": false,
       "lstrip": false,
       "rstrip": false,

tokenizer_config.json

@@ -7,7 +7,7 @@
       "normalized": true,
       "rstrip": false,
       "single_word": false,
-      "special": true
+      "special": false
     },
     "50257": {
       "content": "[aco_0]",
@@ -131130,7 +131130,7 @@
       "special": false
     },
     "66647": {
-      "content": "[spkr_mls_eng_10k_6454]",
+      "content": "[spkr_0]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131138,7 +131138,7 @@
       "special": false
     },
     "66648": {
-      "content": "[spkr_mls_eng_10k_10801]",
+      "content": "[spkr_1]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131146,7 +131146,7 @@
       "special": false
     },
     "66649": {
-      "content": "[spkr_mls_eng_10k_7756]",
+      "content": "[spkr_2]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131154,7 +131154,7 @@
       "special": false
     },
     "66650": {
-      "content": "[spkr_mls_eng_10k_4078]",
+      "content": "[spkr_3]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131162,7 +131162,7 @@
       "special": false
     },
     "66651": {
-      "content": "[spkr_mls_eng_10k_2156]",
+      "content": "[spkr_4]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131170,7 +131170,7 @@
       "special": false
     },
     "66652": {
-      "content": "[spkr_mls_eng_10k_3645]",
+      "content": "[spkr_5]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131178,7 +131178,7 @@
       "special": false
     },
     "66653": {
-      "content": "[spkr_mls_eng_10k_8713]",
+      "content": "[spkr_6]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131186,7 +131186,7 @@
       "special": false
     },
     "66654": {
-      "content": "[spkr_mls_eng_10k_3157]",
+      "content": "[spkr_7]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131194,7 +131194,7 @@
       "special": false
     },
     "66655": {
-      "content": "[spkr_mls_eng_10k_5717]",
+      "content": "[spkr_8]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131202,7 +131202,7 @@
       "special": false
     },
     "66656": {
-      "content": "[spkr_mls_eng_10k_3488]",
+      "content": "[spkr_9]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131210,7 +131210,7 @@
       "special": false
     },
     "66657": {
-      "content": "[spkr_mls_eng_10k_10191]",
+      "content": "[spkr_10]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131218,7 +131218,7 @@
       "special": false
     },
     "66658": {
-      "content": "[spkr_mls_eng_10k_11697]",
+      "content": "[spkr_11]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131226,7 +131226,7 @@
       "special": false
     },
     "66659": {
-      "content": "[spkr_mls_eng_10k_1401]",
+      "content": "[spkr_12]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131234,7 +131234,7 @@
       "special": false
     },
     "66660": {
-      "content": "[spkr_mls_eng_10k_4018]",
+      "content": "[spkr_13]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131242,7 +131242,7 @@
       "special": false
     },
     "66661": {
-      "content": "[spkr_mls_eng_10k_7879]",
+      "content": "[spkr_14]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131250,7 +131250,7 @@
       "special": false
     },
     "66662": {
-      "content": "[spkr_mls_eng_10k_204]",
+      "content": "[spkr_15]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131258,7 +131258,7 @@
       "special": false
     },
     "66663": {
-      "content": "[spkr_mls_eng_10k_8677]",
+      "content": "[spkr_16]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131266,7 +131266,7 @@
       "special": false
     },
     "66664": {
-      "content": "[spkr_mls_eng_10k_107]",
+      "content": "[spkr_17]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131274,7 +131274,7 @@
       "special": false
     },
     "66665": {
-      "content": "[spkr_mls_eng_10k_1259]",
+      "content": "[spkr_18]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131282,7 +131282,7 @@
       "special": false
     },
     "66666": {
-      "content": "[spkr_mls_eng_10k_8138]",
+      "content": "[spkr_19]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131290,7 +131290,7 @@
       "special": false
     },
     "66667": {
-      "content": "[spkr_mls_eng_10k_5968]",
+      "content": "[spkr_20]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131298,7 +131298,7 @@
       "special": false
     },
     "66668": {
-      "content": "[spkr_mls_eng_10k_2607]",
+      "content": "[spkr_21]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131306,7 +131306,7 @@
       "special": false
     },
     "66669": {
-      "content": "[spkr_mls_eng_10k_1724]",
+      "content": "[spkr_22]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131314,7 +131314,7 @@
       "special": false
     },
     "66670": {
-      "content": "[spkr_mls_eng_10k_4788]",
+      "content": "[spkr_23]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131322,7 +131322,7 @@
       "special": false
     },
     "66671": {
-      "content": "[spkr_mls_eng_10k_11667]",
+      "content": "[spkr_24]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131330,7 +131330,7 @@
       "special": false
     },
     "66672": {
-      "content": "[spkr_mls_eng_10k_3681]",
+      "content": "[spkr_25]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131338,7 +131338,7 @@
       "special": false
     },
     "66673": {
-      "content": "[spkr_mls_eng_10k_6671]",
+      "content": "[spkr_26]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131346,7 +131346,7 @@
       "special": false
     },
     "66674": {
-      "content": "[spkr_mls_eng_10k_6670]",
+      "content": "[spkr_27]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131354,7 +131354,7 @@
       "special": false
     },
     "66675": {
-      "content": "[spkr_mls_eng_10k_2234]",
+      "content": "[spkr_28]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131362,7 +131362,7 @@
       "special": false
     },
     "66676": {
-      "content": "[spkr_mls_eng_10k_10244]",
+      "content": "[spkr_29]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131370,7 +131370,7 @@
       "special": false
     },
     "66677": {
-      "content": "[spkr_mls_eng_10k_5985]",
+      "content": "[spkr_30]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131378,7 +131378,7 @@
       "special": false
     },
     "66678": {
-      "content": "[spkr_mls_eng_10k_3699]",
+      "content": "[spkr_31]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131386,7 +131386,7 @@
       "special": false
     },
     "66679": {
-      "content": "[spkr_mls_eng_10k_3003]",
+      "content": "[spkr_32]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131394,7 +131394,7 @@
       "special": false
     },
     "66680": {
-      "content": "[spkr_mls_eng_10k_4174]",
+      "content": "[spkr_33]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131402,7 +131402,7 @@
       "special": false
     },
     "66681": {
-      "content": "[spkr_mls_eng_10k_6660]",
+      "content": "[spkr_34]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131410,7 +131410,7 @@
       "special": false
     },
     "66682": {
-      "content": "[spkr_mls_eng_10k_9017]",
+      "content": "[spkr_35]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131418,7 +131418,7 @@
       "special": false
     },
     "66683": {
-      "content": "[spkr_mls_eng_10k_5304]",
+      "content": "[spkr_36]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131426,7 +131426,7 @@
       "special": false
     },
     "66684": {
-      "content": "[spkr_mls_eng_10k_6687]",
+      "content": "[spkr_37]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131434,7 +131434,7 @@
       "special": false
     },
     "66685": {
-      "content": "[spkr_mls_eng_10k_6497]",
+      "content": "[spkr_38]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131442,7 +131442,7 @@
       "special": false
     },
     "66686": {
-      "content": "[spkr_mls_eng_10k_3595]",
+      "content": "[spkr_39]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131450,7 +131450,7 @@
       "special": false
     },
     "66687": {
-      "content": "[spkr_mls_eng_10k_4064]",
+      "content": "[spkr_40]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131458,7 +131458,7 @@
       "special": false
     },
     "66688": {
-      "content": "[spkr_mls_eng_10k_3657]",
+      "content": "[spkr_41]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131466,7 +131466,7 @@
       "special": false
     },
     "66689": {
-      "content": "[spkr_mls_eng_10k_3967]",
+      "content": "[spkr_42]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131474,7 +131474,7 @@
       "special": false
     },
     "66690": {
-      "content": "[spkr_mls_eng_10k_8011]",
+      "content": "[spkr_43]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131482,7 +131482,7 @@
       "special": false
     },
     "66691": {
-      "content": "[spkr_mls_eng_10k_10463]",
+      "content": "[spkr_44]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131490,7 +131490,7 @@
       "special": false
     },
     "66692": {
-      "content": "[spkr_hifi_tts_9017]",
+      "content": "[spkr_45]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131498,7 +131498,7 @@
       "special": false
     },
     "66693": {
-      "content": "[spkr_hifi_tts_6097]",
+      "content": "[spkr_46]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131506,7 +131506,7 @@
       "special": false
     },
     "66694": {
-      "content": "[spkr_hifi_tts_92]",
+      "content": "[spkr_47]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131514,7 +131514,7 @@
       "special": false
     },
     "66695": {
-      "content": "[spkr_expresso_ex02]",
+      "content": "[spkr_48]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131522,7 +131522,7 @@
       "special": false
     },
     "66696": {
-      "content": "[spkr_expresso_ex03]",
+      "content": "[spkr_49]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131530,7 +131530,7 @@
       "special": false
     },
     "66697": {
-      "content": "[spkr_expresso_ex04]",
+      "content": "[spkr_50]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131538,7 +131538,7 @@
       "special": false
     },
     "66698": {
-      "content": "[spkr_expresso_ex01]",
+      "content": "[spkr_51]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131546,7 +131546,7 @@
       "special": false
     },
     "66699": {
-      "content": "[spkr_jenny_jenny]",
+      "content": "[spkr_52]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131554,7 +131554,7 @@
       "special": false
     },
     "66700": {
-      "content": "[spkr_youtube_webds_hi_kabitaskitchen]",
+      "content": "[spkr_53]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131562,7 +131562,7 @@
       "special": false
     },
     "66701": {
-      "content": "[spkr_youtube_webds_hi_unacademyjee]",
+      "content": "[spkr_54]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131570,7 +131570,7 @@
       "special": false
     },
     "66702": {
-      "content": "[spkr_youtube_webds_hi_technicalguruji]",
+      "content": "[spkr_55]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131578,7 +131578,7 @@
       "special": false
     },
     "66703": {
-      "content": "[spkr_youtube_webds_en_teded]",
+      "content": "[spkr_56]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131586,7 +131586,7 @@
       "special": false
     },
     "66704": {
-      "content": "[spkr_youtube_webds_hi_physicswallah]",
+      "content": "[spkr_57]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131594,7 +131594,7 @@
       "special": false
     },
     "66705": {
-      "content": "[spkr_youtube_webds_en_akshat]",
+      "content": "[spkr_58]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131602,7 +131602,7 @@
       "special": false
     },
     "66706": {
-      "content": "[spkr_youtube_webds_hi_dhruvrathee]",
+      "content": "[spkr_59]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131610,7 +131610,7 @@
       "special": false
     },
     "66707": {
-      "content": "[spkr_youtube_webds_hi_neelimaaudiobooks]",
+      "content": "[spkr_60]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131618,7 +131618,7 @@
       "special": false
     },
     "66708": {
-      "content": "[spkr_youtube_webds_hi_ranveerallahbadia]",
+      "content": "[spkr_61]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131626,7 +131626,7 @@
       "special": false
     },
     "66709": {
-      "content": "[spkr_youtube_webds_hi_hindiaudiobooks]",
+      "content": "[spkr_62]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131634,7 +131634,7 @@
       "special": false
     },
     "66710": {
-      "content": "[spkr_youtube_webds_en_historyofindia]",
+      "content": "[spkr_63]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131642,7 +131642,7 @@
       "special": false
     },
     "66711": {
-      "content": "[spkr_youtube_webds_hi_vivekbindra]",
+      "content": "[spkr_64]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131650,7 +131650,7 @@
       "special": false
     },
     "66712": {
-      "content": "[spkr_youtube_webds_hi_sandeepmaheshwari]",
+      "content": "[spkr_65]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131658,7 +131658,7 @@
       "special": false
     },
     "66713": {
-      "content": "[spkr_youtube_webds_hi_pmmodi]",
+      "content": "[spkr_66]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131666,7 +131666,7 @@
       "special": false
     },
     "66714": {
-      "content": "[spkr_youtube_webds_en_mkbhd]",
+      "content": "[spkr_67]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131674,7 +131674,7 @@
       "special": false
     },
     "66715": {
-      "content": "[spkr_youtube_webds_en_secondhandstories]",
+      "content": "[spkr_68]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131682,7 +131682,7 @@
       "special": false
     },
     "66716": {
-      "content": "[spkr_youtube_webds_en_storiesofmahabharatha]",
+      "content": "[spkr_69]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131690,7 +131690,7 @@
       "special": false
     },
     "66717": {
-      "content": "[spkr_youtube_webds_hi_a2motivation]",
+      "content": "[spkr_70]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131698,7 +131698,7 @@
       "special": false
     },
     "66718": {
-      "content": "[spkr_youtube_webds_hi_mrbeast]",
+      "content": "[spkr_71]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131706,7 +131706,7 @@
       "special": false
     },
     "66719": {
-      "content": "[spkr_youtube_webds_hi_akshat]",
+      "content": "[spkr_72]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131714,7 +131714,7 @@
       "special": false
     },
     "66720": {
-      "content": "[spkr_shrutilipi_hi_shrutilipi]",
+      "content": "[spkr_73]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131722,7 +131722,7 @@
       "special": false
     },
     "66721": {
-      "content": "[spkr_youtube_webds_en_derekperkins]",
+      "content": "[spkr_74]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131730,7 +131730,7 @@
       "special": false
     },
     "66722": {
-      "content": "[spkr_youtube_webds_en_mukesh]",
+      "content": "[spkr_75]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131738,7 +131738,7 @@
       "special": false
     },
     "66723": {
-      "content": "[spkr_youtube_webds_en_attenborough]",
+      "content": "[spkr_76]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131746,7 +131746,7 @@
       "special": false
     },
     "66724": {
-      "content": "[spkr_youtube_webds_hi_warikoo]",
+      "content": "[spkr_77]",
       "lstrip": false,
       "normalized": true,
       "rstrip": false,
@@ -131755,7 +131755,7 @@
     }
   },
   "bos_token": "<|endoftext|>",
-  "clean_up_tokenization_spaces": true,
+  "clean_up_tokenization_spaces": false,
   "eos_token": "<|endoftext|>",
   "model_max_length": 1024,
   "tokenizer_class": "GPT2Tokenizer",