Hugging Face's logo

hugohrban
/
progen2-small-mix7-bidi

Text Generation
Transformers
Safetensors
progen
protein
custom_code
Model card Files Community
hugohrban commited on
Commit
6c45e77
•
1 Parent(s): 12e73a0

Create tokenizer.json

Browse files
Files changed (1) hide show
  1. tokenizer.json +167 -0
tokenizer.json ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "version": "1.0",
3
+ "truncation": {
4
+ "max_length": 1024,
5
+ "strategy": "LongestFirst",
6
+ "stride": 0
7
+ },
8
+ "padding": {
9
+ "strategy": {
10
+ "Fixed": 1024
11
+ },
12
+ "direction": "Right",
13
+ "pad_to_multiple_of": null,
14
+ "pad_id": 0,
15
+ "pad_type_id": 0,
16
+ "pad_token": "<|pad|>"
17
+ },
18
+ "added_tokens": [
19
+ {
20
+ "id": 0,
21
+ "special": true,
22
+ "content": "<|pad|>",
23
+ "single_word": false,
24
+ "lstrip": false,
25
+ "rstrip": false,
26
+ "normalized": false
27
+ },
28
+ {
29
+ "id": 1,
30
+ "special": true,
31
+ "content": "<|bos|>",
32
+ "single_word": false,
33
+ "lstrip": false,
34
+ "rstrip": false,
35
+ "normalized": false
36
+ },
37
+ {
38
+ "id": 2,
39
+ "special": true,
40
+ "content": "<|eos|>",
41
+ "single_word": false,
42
+ "lstrip": false,
43
+ "rstrip": false,
44
+ "normalized": false
45
+ },
46
+ {
47
+ "id": 30,
48
+ "special": false,
49
+ "content": "<|pf00002|>",
50
+ "single_word": false,
51
+ "lstrip": false,
52
+ "rstrip": false,
53
+ "normalized": true
54
+ },
55
+ {
56
+ "id": 31,
57
+ "special": false,
58
+ "content": "<|pf00042|>",
59
+ "single_word": false,
60
+ "lstrip": false,
61
+ "rstrip": false,
62
+ "normalized": true
63
+ },
64
+ {
65
+ "id": 32,
66
+ "special": false,
67
+ "content": "<|pf00125|>",
68
+ "single_word": false,
69
+ "lstrip": false,
70
+ "rstrip": false,
71
+ "normalized": true
72
+ },
73
+ {
74
+ "id": 33,
75
+ "special": false,
76
+ "content": "<|pf00127|>",
77
+ "single_word": false,
78
+ "lstrip": false,
79
+ "rstrip": false,
80
+ "normalized": true
81
+ },
82
+ {
83
+ "id": 34,
84
+ "special": false,
85
+ "content": "<|pf00257|>",
86
+ "single_word": false,
87
+ "lstrip": false,
88
+ "rstrip": false,
89
+ "normalized": true
90
+ },
91
+ {
92
+ "id": 35,
93
+ "special": false,
94
+ "content": "<|pf00262|>",
95
+ "single_word": false,
96
+ "lstrip": false,
97
+ "rstrip": false,
98
+ "normalized": true
99
+ },
100
+ {
101
+ "id": 36,
102
+ "special": false,
103
+ "content": "<|pf03668|>",
104
+ "single_word": false,
105
+ "lstrip": false,
106
+ "rstrip": false,
107
+ "normalized": true
108
+ }
109
+ ],
110
+ "normalizer": null,
111
+ "pre_tokenizer": {
112
+ "type": "ByteLevel",
113
+ "add_prefix_space": false,
114
+ "trim_offsets": true
115
+ },
116
+ "post_processor": {
117
+ "type": "ByteLevel",
118
+ "add_prefix_space": true,
119
+ "trim_offsets": true
120
+ },
121
+ "decoder": {
122
+ "type": "ByteLevel",
123
+ "add_prefix_space": true,
124
+ "trim_offsets": true
125
+ },
126
+ "model": {
127
+ "type": "BPE",
128
+ "dropout": null,
129
+ "unk_token": null,
130
+ "continuing_subword_prefix": null,
131
+ "end_of_word_suffix": null,
132
+ "fuse_unk": false,
133
+ "vocab": {
134
+ "<|pad|>": 0,
135
+ "<|bos|>": 1,
136
+ "<|eos|>": 2,
137
+ "1": 3,
138
+ "2": 4,
139
+ "A": 5,
140
+ "B": 6,
141
+ "C": 7,
142
+ "D": 8,
143
+ "E": 9,
144
+ "F": 10,
145
+ "G": 11,
146
+ "H": 12,
147
+ "I": 13,
148
+ "K": 14,
149
+ "L": 15,
150
+ "M": 16,
151
+ "N": 17,
152
+ "O": 18,
153
+ "P": 19,
154
+ "Q": 20,
155
+ "R": 21,
156
+ "S": 22,
157
+ "T": 23,
158
+ "U": 24,
159
+ "V": 25,
160
+ "W": 26,
161
+ "X": 27,
162
+ "Y": 28,
163
+ "Z": 29
164
+ },
165
+ "merges": []
166
+ }
167
+ }