Upload tokenizer
d677f70
verified
|
{ |
|
"<am>": 15000, |
|
"<extra_id_0>": 15099, |
|
"<extra_id_10>": 15089, |
|
"<extra_id_11>": 15088, |
|
"<extra_id_12>": 15087, |
|
"<extra_id_13>": 15086, |
|
"<extra_id_14>": 15085, |
|
"<extra_id_15>": 15084, |
|
"<extra_id_16>": 15083, |
|
"<extra_id_17>": 15082, |
|
"<extra_id_18>": 15081, |
|
"<extra_id_19>": 15080, |
|
"<extra_id_1>": 15098, |
|
"<extra_id_20>": 15079, |
|
"<extra_id_21>": 15078, |
|
"<extra_id_22>": 15077, |
|
"<extra_id_23>": 15076, |
|
"<extra_id_24>": 15075, |
|
"<extra_id_25>": 15074, |
|
"<extra_id_26>": 15073, |
|
"<extra_id_27>": 15072, |
|
"<extra_id_28>": 15071, |
|
"<extra_id_29>": 15070, |
|
"<extra_id_2>": 15097, |
|
"<extra_id_30>": 15069, |
|
"<extra_id_31>": 15068, |
|
"<extra_id_32>": 15067, |
|
"<extra_id_33>": 15066, |
|
"<extra_id_34>": 15065, |
|
"<extra_id_35>": 15064, |
|
"<extra_id_36>": 15063, |
|
"<extra_id_37>": 15062, |
|
"<extra_id_38>": 15061, |
|
"<extra_id_39>": 15060, |
|
"<extra_id_3>": 15096, |
|
"<extra_id_40>": 15059, |
|
"<extra_id_41>": 15058, |
|
"<extra_id_42>": 15057, |
|
"<extra_id_43>": 15056, |
|
"<extra_id_44>": 15055, |
|
"<extra_id_45>": 15054, |
|
"<extra_id_46>": 15053, |
|
"<extra_id_47>": 15052, |
|
"<extra_id_48>": 15051, |
|
"<extra_id_49>": 15050, |
|
"<extra_id_4>": 15095, |
|
"<extra_id_50>": 15049, |
|
"<extra_id_51>": 15048, |
|
"<extra_id_52>": 15047, |
|
"<extra_id_53>": 15046, |
|
"<extra_id_54>": 15045, |
|
"<extra_id_55>": 15044, |
|
"<extra_id_56>": 15043, |
|
"<extra_id_57>": 15042, |
|
"<extra_id_58>": 15041, |
|
"<extra_id_59>": 15040, |
|
"<extra_id_5>": 15094, |
|
"<extra_id_60>": 15039, |
|
"<extra_id_61>": 15038, |
|
"<extra_id_62>": 15037, |
|
"<extra_id_63>": 15036, |
|
"<extra_id_64>": 15035, |
|
"<extra_id_65>": 15034, |
|
"<extra_id_66>": 15033, |
|
"<extra_id_67>": 15032, |
|
"<extra_id_68>": 15031, |
|
"<extra_id_69>": 15030, |
|
"<extra_id_6>": 15093, |
|
"<extra_id_70>": 15029, |
|
"<extra_id_71>": 15028, |
|
"<extra_id_72>": 15027, |
|
"<extra_id_73>": 15026, |
|
"<extra_id_74>": 15025, |
|
"<extra_id_75>": 15024, |
|
"<extra_id_76>": 15023, |
|
"<extra_id_77>": 15022, |
|
"<extra_id_78>": 15021, |
|
"<extra_id_79>": 15020, |
|
"<extra_id_7>": 15092, |
|
"<extra_id_80>": 15019, |
|
"<extra_id_81>": 15018, |
|
"<extra_id_82>": 15017, |
|
"<extra_id_83>": 15016, |
|
"<extra_id_84>": 15015, |
|
"<extra_id_85>": 15014, |
|
"<extra_id_86>": 15013, |
|
"<extra_id_87>": 15012, |
|
"<extra_id_88>": 15011, |
|
"<extra_id_89>": 15010, |
|
"<extra_id_8>": 15091, |
|
"<extra_id_90>": 15009, |
|
"<extra_id_91>": 15008, |
|
"<extra_id_92>": 15007, |
|
"<extra_id_93>": 15006, |
|
"<extra_id_94>": 15005, |
|
"<extra_id_95>": 15004, |
|
"<extra_id_9>": 15090, |
|
"<geez>": 15002, |
|
"<tig>": 15001, |
|
"<tigre>": 15003 |
|
} |
|
|