Upload tokenizer

Browse files

Files changed (3) hide show

special_tokens_map.json +7 -0
tokenizer_config.json +58 -0
vocab.txt +1024 -0

special_tokens_map.json ADDED Viewed

	@@ -0,0 +1,7 @@

+{
+  "cls_token": "[CLS]",
+  "mask_token": "[MASK]",
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "unk_token": "[UNK]"
+}

tokenizer_config.json ADDED Viewed

	@@ -0,0 +1,58 @@

+{
+  "added_tokens_decoder": {
+    "0": {
+      "content": "[PAD]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "[UNK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "[CLS]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "[SEP]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "do_basic_tokenize": true,
+  "do_lower_case": true,
+  "extra_special_tokens": {},
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "never_split": null,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "strip_accents": null,
+  "tokenize_chinese_chars": true,
+  "tokenizer_class": "BertTokenizer",
+  "unk_token": "[UNK]"
+}

vocab.txt ADDED Viewed

	@@ -0,0 +1,1024 @@

+[PAD]
+[UNK]
+[CLS]
+[SEP]
+[MASK]
+(
+)
+,
+-
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
+?
+[
+]
+a
+b
+c
+d
+e
+f
+g
+h
+i
+k
+l
+m
+n
+o
+p
+q
+r
+s
+t
+u
+v
+w
+x
+y
+z
+{
+}
+##d
+##a
+##r
+##f
+##u
+##3
+##c
+##e
+##n
+##m
+##7
+##g
+##l
+##h
+##p
+##o
+##4
+##i
+##2
+##s
+##6
+##b
+##t
+##1
+##x
+##y
+##5
+##9
+##v
+##z
+##8
+##w
+b1
+##ac
+##lc
+glc
+##nac
+a1
+glcnac
+##al
+gal
+##an
+man
+##uc
+fuc
+a2
+##eu
+neu
+neu5
+neu5ac
+galnac
+rh
+rha
+##gc
+neu5gc
+##yl
+xyl
+##me
+glca
+kd
+glcn
+##he
+ol
+##hep
+##dm
+##anhep
+##dmanhep
+##ra
+ara
+kdo
+ldmanhep
+##6s
+##op
+galf
+araf
+##2s
+gala
+##3me
+qu
+qui
+he
+##ome
+hex
+##ro
+kdn
+##ru
+fru
+##et
+##3s
+fruf
+glcnac6s
+##6p
+glc1
+##er
+##etn
+##os
+##cer
+ri
+rib
+##do
+ido
+##6ac
+glc1cer
+##4p
+##1p
+fucnac
+idoa
+b2
+##4s
+quinac
+##2ac
+glcn2s
+##4ac
+##yr
+hexnac
+mannac
+##oac
+##3ac
+##4n
+##nh
+anh
+##dro
+##ydro
+anhydro
+gal3me
+ldmanhepop
+##2me
+##gro
+6d
+idoa2s
+##6me
+##4me
+ribf
+man6p
+ddmanhep
+gal3s
+##tal
+6dtal
+##ch
+##cho
+##1n
+glcn2s6s
+galnac4s
+##2nac
+##3nac
+galn
+gal6s
+##5ac
+##fo
+galos
+glcn1p
+##petn
+##bu
+##but
+glcn4p
+glc2s
+glcnacos
+glc6ac
+di
+manome
+dig
+manop
+##ul
+##3n
+rha4n
+##ala
+mana
+mannaca
+gul
+galnaca
+##am
+man3me
+##3s6s
+glcnac1n
+##myr
+##4pyr
+##3p
+glcnacop
+glcnac6p
+##6n
+xylf
+ribop
+##9ac
+kdoop
+##3ac4ac
+galop
+rha2ac
+##5p
+##ol
+##1me
+ldmanhepopetn
+##7ac
+##2p
+##hex
+ps
+##be
+abe
+pse
+##cm
+glcop
+##2me3me
+al
+##4nac
+man1p
+##ic
+rhaome
+glcn2
+rib5p
+rha4nfo
+rhanac
+gal4s
+glc2ac
+##2nac3nac
+glca4me
+##ns
+##ocm
+ins
+##3s4s
+galnac6s
+##5ac7ac
+glcnacopcho
+neu5ac9ac
+##nic
+rha2me
+glcnaca
+manos
+galnacos
+fuc2me
+dig3me
+glcome
+rha3me
+##hr
+##en
+glc6me
+oli
+man6petn
+gal3p
+quin
+onic
+glc6s
+gal6p
+glcnac1p
+neu4ac
+gala2nac
+glcn2s3s6s
+glc6p
+manopetn
+ap
+ty
+fucome
+api
+tyv
+fuc2s
+##3ac4ac6ac
+##eg
+rhaoac
+##co
+##lac
+##thr
+gula
+##ly
+apif
+galnacop
+glca3s
+glcnac6pcho
+##cl
+ribf1n
+gulnac
+gal2ac
+glc2s3s6s
+gal4pyr
+fucn
+qui3nac
+##ara
+##3myr
+manf
+glca6me
+qui4n
+col
+##ar
+ldmanhepoppetn
+neu4ac5ac
+##ep
+hexa
+hexome
+##aep
+##6aep
+ara4n
+##arahex
+glcnac6ac
+all
+##au
+6dal
+mana2nac3nac
+glcnac6petn
+galoac
+galnala
+mu
+##pgro
+##6pyr
+gal4pyr6pyr
+mur
+glcn2myr
+##lau
+##3s4s6s
+alt
+glcnacoac
+##3ac6ac
+manf1me
+4d
+murnac
+glca2s
+glcn3s
+##fer
+##6petn
+##nbut
+rha3ac
+gal2s
+ldmanhep4p
+fucnacn
+ko
+galopgro
+glc3me
+glcnacn
+glcaome
+fucnac4n
+6dalt
+##4ac6ac
+6dtaloac
+##olac
+glc2me3me
+manoac
+6darahex
+gal4me
+kdo4p
+gulnaca
+leg
+##vac
+rha4ac
+fruf1
+galome
+man2ac
+pen
+##1cer
+man6me
+galaome
+pse5ac7ac
+gala6n
+##3petn
+galopyr
+##1pp
+6dtal2ac
+##2nac3n
+par
+6darahexome
+gal2me
+galf2ac
+##bz
+gal6me
+glc2s3s4s6s
+##anam
+man6ac
+kdooppetn
+galnacopcho
+##3lac
+glc3s
+glc4pyr
+ldmanhepopocm
+galnacome
+glc1p
+4dthr
+##4en
+galnac6p
+ly
+##3et
+glc2
+glcos
+##opn
+lyx
+a6
+tal
+man6s
+rha4nac
+gala6me
+glcn3n
+quinac4nac
+galnacaoac
+4dthrhex
+gal1cer
+qui3n
+idoa2s3et
+##2n
+glc2p
+fucoac
+##4s6s
+gala2nac3ac
+##dco
+glcnacan
+glcnme
+##4nbut
+digome
+##alaac
+kdoopetn
+glcnaca3nac
+##um
+gala6
+##4me6s
+glcn1p2
+##coum
+glcn2myr3myr
+bac
+er
+mann
+##gly
+glc3ac
+man2
+galnac3s
+##4pgro
+mannacanam
+##gron
+psenac
+b6
+##el
+##obz
+gal6ac
+rha3petn
+rha4nbut
+mannaca3nac
+##ole
+##2pam
+gal3pyr
+ery
+##eleg
+##dg
+##4lac
+##1fer
+glcnac3ac
+man4ac
+ldmanhep6petn
+galnac4s6s
+glcopyr
+neu9ac
+##8me
+man4me
+fuc3n
+##2ac3ac
+glc2s4s6s
+glc2ac3ac6ac
+6dmanhep
+ar
+##a4en
+##3dco
+glcnac6
+ara1p
+galaoac
+fuc2s4s
+##lyx
+bac2ac
+gal3pyr4pyr
+##te
+##6sn
+glc2s6sn
+ul
+##1vac
+man1
+man2p
+##opgro
+##5ac7
+4dthrhexa4en
+ulos
+8eleg
+##6et
+fuc2me3me
+glc1fer
+mana2nac3n
+glc6pgro
+gal1p
+man2me
+glcn2pam
+araopn
+##opetn
+aran
+##ser
+##alhep
+fucf
+neu5acoac
+xylome
+glcaos
+rib1p
+quinacnbut
+glc2me3me6me
+3d
+ddg
+##lu
+##6cl
+glc4me
+glc3ac4ac6ac
+neu1n
+neu5ac7ac
+galnacan
+galnfo
+su
+##3c
+##4cl
+fuc3me
+glca6
+glca2nac3nac
+glcn2me
+fucnacoac
+idoa2s3me
+6dtal3me
+oli3nac
+glcnac6gc
+ara1p4n
+sug
+##3lau
+##8s
+rha4me
+glca2me3me
+glcn6ac
+##3me4me6s
+xlu
+##obut
+##1gro
+glcpgro
+glcn6aep
+gala1p
+6dtalome
+glcopgro
+galnac6pcho
+ddgalhep
+xluf
+glcgro
+gala6ala
+glcnac4p
+fuc4s
+##opyr
+gala3me
+glc2s3s
+galnaca6n
+abe2ac
+gala2nac6n
+fucnam
+leg5ac7ac
+penf
+bac2ac4ac
+11
+##7petn
+##6f
+##ys
+gal4p
+gal2p
+fuc3s
+fucos
+neu5ac9
+rhagro
+neu5gc8me
+glc1n
+glc1me
+glc1gro
+6dg
+mana6me
+pse5ac7
+gal2ac3ac4ac6ac
+##or
+##1coum
+glc4s
+man3p
+man1me
+ldmanhepocm
+qui2me
+frufobz
+##ama
+rha2ac4ac
+galnacaoacn
+8eleg5ac7ac
+thr
+##rop
+glcnacgro
+glcnac3lac
+man6aep
+glca6ac
+hexnacos
+mana2nac
+quinbut
+glcnac1pp
+altnac
+glcn3n4p
+glcnmeocm
+glcn1p2myr
+man2petn
+##lyxhep
+thre
+sor
+##oetn
+##omyr
+glcnac3ac6ac
+##alys
+gal4cl
+fuc2ac
+rha2me3me
+glc1ole
+##ery
+##lys
+man1s
+##nfo
+galnac3ac
+rha3ac4ac
+xyl3me
+quinac4nbut
+mannac1p
+##alafo
+glcopn
+fuc2me4ac
+gal3pgro
+##aric
+fruf1cl
+ulosaric
+3dlyxhep
+neu1n5ac
+galnfoa
+##3cme
+6dgul
+altnaca
+dl
+##6nac
+##8ac
+glc4ac
+glcnac1me
+gal2
+rha1p
+glca6s
+glca2me
+glcn6s
+hep
+##2ac3ac4ac
+rha2ac3ac4ac
+gula2nac3nac
+gal2ac4ac6ac
+qui4nfo
+##lau3dco
+6dtaloacome
+glc2me3me4me
+kdo4ppetn
+psenacnbut
+##xyl
+gal6aep
+fuc4ac
+galnac1p
+ldmanhep6p
+galagron
+glcn2s3s
+glc2ac3ac4ac6ac
+quinalaac
+allnac
+ldmanhep4ppetn
+fucnacnme
+##xylhex
+1d
+yer
+glcnac3me
+galfoac
+manopyr
+##ulo
+##3nam
+##3pgro
+glcn2myr3myr4p
+fuc3nbut
+gal2pgro
+##ero
+##nam
+##4f
+xyl2me
+##1pam
+##groa
+glc2s3s4s
+glc2s3me4me6s
+6dtal2ac3me
+glc1pam
+pse5ac7fo
+##br
+glcolac
+glcnac3s
+man2s
+neu5ac8s
+glca2nac
+glca2nac3n
+glcn3s6s
+ldmanhep3petn
+galfnac
+gala4ac
+gala2n
+fucnac4ac
+fucnac4nac
+glcn2ste
+6dxylhex
+##5ac6ac
+gal6petn
+glca2s3me
+6daltf
+fucoacome
+glcnacanalafo
+neu5ac9nac
+qui2me4n
+##nme
+##7p
+glcan
+glc6n
+glcnac4me
+gal4ac
+gal3ac
+man4pyr
+rhan
+rhaobut
+xyl3ac
+glcn3ac4ac6ac
+gala2s
+glc1pgro
+galnaca3ac
+galopcho
+pse5
+oli4me
+galnacopgro
+6dall
+legnac
+fruf1ac
+6dgulhep
+1dery
+##fero
+##onic
+##4ulo
+##2vac
+##5gc
+glc4ac6ac
+glcnac4lac
+gal3
+neu1me
+rha3s
+glca2p
+glcaolac
+6dhex
+6dtal2me
+glc2ac3ac
+rha2me4n
+rha2me4me
+rha2me4nbut
+fruf1fero
+ulosonic
+gal1pgro
+glcn2pam3myr
+##3lau4p
+sorf
+glca2pyr
+dh
+##on
+##2ala
+##5s
+##5nac
+glcnac1
+gal1me
+gal6f
+man3s
+fuc4n
+galnac6ac
+neu5gc8s
+xyl4s
+glcn6petn
+glcnama
+glcnomyr
+kdo5p
+qui4nac
+idonac
+mannaca4ac
+mannaca3nam
+quinala
+##thrac
+alta
+glc3me6me
+qui3nbut
+frufobz1fer
+fruf1ferobz
+dha
+##mar
+##2myr
+##9me
+glc3p
+glcnacome
+man3ac
+man4lac
+galnac3me
+galnac1pp
+galnac3pgro
+xyl1n
+glca2n
+araome
+kdo7petn
+galf5ac6ac
+##et4s
+##et3s4s
+glcn2s3me4me6s
+mannacan
+gal3s6s
+glc6ac1fer
+galopetn
+glcn2vac
+glca4me6n
+rha3me4me
+gal6pcho
+fucnbut
+glcn2myr3n
+4darahex
+glc2f
+glcnme1vac
+psenacnfo
+glc1fer6ac
+neu5ac7ac9ac
+##lau3dco4p
+gal3f
+##et4s6et
+##et3s4s6et
+2d
+##non
+##7fo
+##glc
+##6i
+##6gly
+glc3s6s
+glcnac4pgro
+galgro
+galalys
+##an4en
+man4n
+man4ac6ac
+fuc3s4s
+rhaop
+neu5gc9ac
+xyl4ac
+glcns
+kdof
+ldmanhep7petn
+fruf4ac
+frufoac
+quinac4n
+mannac2nac
+6dtalnac
+##3pyr
+rhanac3nac
+neu4ac5gc
+gulnacanam
+gal2s3s4s6s
+6dalthep
+fruf1coum
+neu1me5ac
+5d
+##3fo
+glcnac4pyr
+glcnac6aep
+gal6n
+fuc3ac
+fuc3ac4ac
+galnac6me
+rhaos
+rhaolac
+rha1gro
+neu5gcos
+glca3lac
+glcagron
+glcn6p
+glcnbut
+glcncm
+kdo8
+kdo1me
+ldmanhepoac
+##6ser
+##opcho
+galf6ac
+##2suc
+galaop
+galaala
+rib3p
+##oacop
+gal3s4s
+galn2suc
+glcn1ppetn
+mannaca2nac
+glcn2lau3dco4p
+glca4me6me
+glc3me4pyr
+gulnaca3nac
+6dtal2ac4ac
+gala6lys
+glcn1p2lau
+man1etn
+fuc2me3me4me
+man2me4me
+galnfoan
+fucosome
+glc1ole6ac
+glca2nac3n6n
+glca2pyr3pyr
+9dg
+b3
+##as
+##pcho
+##pen
+##oco
+##in
+glc3lac
+##nac4ulo
+galan
+galolac
+fuc1p
+xyl4me
+xyl2p
+ara2ac
+galf6p
+quiome
+fruf4ac6ac
+glc1coum
+quinacn
+hexnacopcho
+mannac6petn
+6dtal4ac
+glcn1petn
+dig3ac
+manopcho
+##alabut
+##am7
+kdoopn
+glcopetn
+glcopcho
+glcn2ala
+glcnaca3n
+quinfo
+glcnac1prop
+glc6pcho
+qui4nalabut
+manf1me5p
+parf
+4dthrhexan4en
+frufobz1coum
+glcn1p2myr3myr
+glc1pam6ac
+6dxylhexnac4ulo
+pse5am7
+2darahex
+quinac4nhex
+fruf1coumobz
+##asp
+se
+##3gc
+##oc
+##4gc
+##ido
+glc3s4s
+glc6nac
+glc6i
+glcnac6me
+glcnac3nac
+glcnac3s6s
+glcnac3ac4ac6ac
+##alt
+gal1pp
+##me5ac
+glca4lac
+glcaser
+glcn1n
+ldmanhep2p
+galf5p
+araf1fer
+##2ste
+gala2ac
+gala6ser
+##3meopn
+qui6s