BioTokenizer-BFD-UNI-200 / tokenizer.json
dotan1111's picture
Upload 2 files
8716dc4
raw
history blame contribute delete
No virus
12.1 kB
{
"version": "1.0",
"truncation": null,
"padding": null,
"added_tokens": [
{
"id": 0,
"content": "<UNK>",
"single_word": false,
"lstrip": false,
"rstrip": false,
"normalized": false,
"special": true
}
],
"normalizer": {
"type": "Lowercase"
},
"pre_tokenizer": {
"type": "Whitespace"
},
"post_processor": null,
"decoder": null,
"model": {
"type": "Unigram",
"unk_id": 0,
"vocab": [
[
"<UNK>",
0.0
],
[
"y",
-3.4247079552329573
],
[
"h",
-3.4912839749476063
],
[
"a",
-3.4928119486871907
],
[
"q",
-3.54294177673021
],
[
"m",
-3.5727061235339868
],
[
"n",
-3.6861399265992674
],
[
"l",
-3.6902237446793507
],
[
"r",
-3.709010589966521
],
[
"f",
-3.764879814809312
],
[
"v",
-3.814862110364423
],
[
"g",
-3.8544927179040798
],
[
"p",
-3.879554414552608
],
[
"d",
-3.9147195920056497
],
[
"t",
-3.9218713965741117
],
[
"w",
-3.9222956068813346
],
[
"s",
-3.9578207791967905
],
[
"e",
-3.9672879466924833
],
[
"c",
-3.9846198307959853
],
[
"k",
-3.9851137367724085
],
[
"i",
-4.193013703452998
],
[
"aa",
-4.436907022724377
],
[
"la",
-4.763564440735472
],
[
"al",
-4.858898617206826
],
[
"ag",
-4.871741733505836
],
[
"ll",
-4.899529457955385
],
[
"rr",
-4.9773160337094495
],
[
"gg",
-5.014989277014378
],
[
"va",
-5.109713869003169
],
[
"ar",
-5.138860598222161
],
[
"av",
-5.139874127757343
],
[
"rl",
-5.176546293029192
],
[
"ga",
-5.180699752082347
],
[
"ra",
-5.2000038343828034
],
[
"lg",
-5.2027465134290285
],
[
"lv",
-5.256125973713518
],
[
"vl",
-5.271594558844297
],
[
"gl",
-5.30076007905488
],
[
"vv",
-5.317697085475906
],
[
"lr",
-5.322075731145462
],
[
"pa",
-5.355656845053176
],
[
"gr",
-5.375270652971334
],
[
"el",
-5.418090097414771
],
[
"sg",
-5.430395962674904
],
[
"gv",
-5.436901106950042
],
[
"ls",
-5.4458744055610495
],
[
"ss",
-5.448111897232318
],
[
"ae",
-5.461435663548462
],
[
"lp",
-5.475994138455475
],
[
"as",
-5.476134811820398
],
[
"ea",
-5.485108044375101
],
[
"sa",
-5.500928466436452
],
[
"ld",
-5.508944898925408
],
[
"sl",
-5.514725532043448
],
[
"vg",
-5.518212225765366
],
[
"dl",
-5.523790559991012
],
[
"ad",
-5.529423509798146
],
[
"rg",
-5.535085863481557
],
[
"da",
-5.536839902357814
],
[
"er",
-5.53952607664727
],
[
"at",
-5.5447440770682555
],
[
"ia",
-5.562486066774282
],
[
"ta",
-5.562710078562095
],
[
"ap",
-5.576109295337275
],
[
"pg",
-5.5889310432053705
],
[
"gs",
-5.613078970833071
],
[
"lt",
-5.6256524683818725
],
[
"tl",
-5.625679592438182
],
[
"dg",
-5.6321727208112105
],
[
"tg",
-5.637149672051338
],
[
"rs",
-5.642993846504739
],
[
"le",
-5.646009973570775
],
[
"rv",
-5.65593574252734
],
[
"rp",
-5.690244716736588
],
[
"pp",
-5.699015023501083
],
[
"vr",
-5.713330922855734
],
[
"gt",
-5.724597713519122
],
[
"x",
-5.749277420153369
],
[
"vt",
-5.773882873295735
],
[
"gd",
-5.777222073452883
],
[
"ve",
-5.786282370291476
],
[
"ai",
-5.792878037369006
],
[
"vd",
-5.794675365344499
],
[
"re",
-5.79541316342204
],
[
"ee",
-5.795741716638281
],
[
"vs",
-5.7985576242235215
],
[
"dv",
-5.803070571126753
],
[
"tv",
-5.8066339742199276
],
[
"ge",
-5.816848413000429
],
[
"pl",
-5.818667195662142
],
[
"ig",
-5.8351234290286556
],
[
"pv",
-5.85521794340395
],
[
"sp",
-5.856237812350054
],
[
"sr",
-5.863093598657972
],
[
"dr",
-5.865626231550317
],
[
"ev",
-5.867222604064999
],
[
"sv",
-5.8821181233543385
],
[
"tt",
-5.888419823057225
],
[
"vp",
-5.893708098390535
],
[
"rd",
-5.902907686889787
],
[
"st",
-5.913895223643719
],
[
"tp",
-5.925474310054826
],
[
"ts",
-5.935259750541029
],
[
"iv",
-5.936481760515951
],
[
"ps",
-5.957155519695867
],
[
"dp",
-5.959226050226507
],
[
"gi",
-5.959289362288203
],
[
"de",
-5.97130066143788
],
[
"pr",
-5.979541473966986
],
[
"li",
-5.995759213159367
],
[
"il",
-6.010252230503628
],
[
"kl",
-6.022115205105928
],
[
"rt",
-6.032960790021843
],
[
"eg",
-6.039348346316448
],
[
"ei",
-6.040448664946604
],
[
"ql",
-6.041916049879731
],
[
"id",
-6.042274769013373
],
[
"pd",
-6.045243783203977
],
[
"gp",
-6.049733643791777
],
[
"qa",
-6.050023019599072
],
[
"dd",
-6.055774660433624
],
[
"pe",
-6.084564384204274
],
[
"ri",
-6.088601223547604
],
[
"is",
-6.100326314999879
],
[
"ie",
-6.109111961781512
],
[
"ka",
-6.121224426031684
],
[
"lf",
-6.131569198939003
],
[
"kk",
-6.135328802275913
],
[
"vi",
-6.145707458064743
],
[
"fg",
-6.147380503177732
],
[
"fl",
-6.152282406338413
],
[
"fa",
-6.182936168452729
],
[
"pt",
-6.183826012732906
],
[
"qr",
-6.195343435788958
],
[
"sd",
-6.207739044792362
],
[
"af",
-6.233639395340617
],
[
"et",
-6.235456716778566
],
[
"gf",
-6.243328452420121
],
[
"si",
-6.250022710702352
],
[
"ek",
-6.2547582409569
],
[
"tr",
-6.258601043179839
],
[
"it",
-6.265827079759829
],
[
"ti",
-6.2667690455532075
],
[
"lk",
-6.266837815225498
],
[
"gk",
-6.2791883478354045
],
[
"aq",
-6.285028901209582
],
[
"ep",
-6.294333027943258
],
[
"ed",
-6.295524393472979
],
[
"ke",
-6.328320481376119
],
[
"nl",
-6.333964002663926
],
[
"es",
-6.335631844356625
],
[
"se",
-6.345943093920235
],
[
"ak",
-6.3579406834980965
],
[
"td",
-6.376912272540318
],
[
"kt",
-6.381749192026097
],
[
"kv",
-6.383192653367846
],
[
"lq",
-6.386738517935324
],
[
"fv",
-6.39360783803135
],
[
"qv",
-6.396066490337569
],
[
"sf",
-6.39910526635248
],
[
"fs",
-6.40212185272803
],
[
"rf",
-6.412047799425322
],
[
"ks",
-6.4128110020819555
],
[
"ng",
-6.421427777161963
],
[
"ki",
-6.422202910211933
],
[
"ip",
-6.430876724365621
],
[
"rq",
-6.435834150272331
],
[
"fd",
-6.446507064108266
],
[
"ir",
-6.449152590874585
],
[
"ln",
-6.453108969541907
],
[
"kr",
-6.457387963642308
],
[
"vf",
-6.463731630061847
],
[
"eq",
-6.46524591662215
],
[
"qp",
-6.480794924169258
],
[
"di",
-6.509616735402286
],
[
"kp",
-6.510059533681824
],
[
"na",
-6.51587869796394
],
[
"ds",
-6.523174036007418
],
[
"kg",
-6.534881877135531
],
[
"np",
-6.550937681060169
],
[
"gn",
-6.566725084409789
],
[
"gq",
-6.567082940116801
],
[
"kd",
-6.568698331293467
],
[
"qg",
-6.569785852284269
],
[
"te",
-6.579979481653046
],
[
"dt",
-6.582071618443225
],
[
"rk",
-6.594085374063781
],
[
"qs",
-6.606515808555127
],
[
"hl",
-6.612070459132525
],
[
"ii",
-6.622224788525211
],
[
"ft",
-6.6381275899184775
],
[
"nv",
-6.661901139647316
],
[
"df",
-6.667420037978074
],
[
"tf",
-6.676255399402093
],
[
"pi",
-6.717126026714817
],
[
"sn",
-6.723550604685874
],
[
"u",
-17.611299956393843
],
[
"b",
-17.811943459429884
],
[
"z",
-19.052756709660283
],
[
"o",
-20.50275670966017
]
]
}
}