Hugging Face's logo

Echo9Zulu
/
dolphincoder-starcoder2-15b-int8-ns-ov

OpenVINO
starcoder2
Model card Files Community
Echo9Zulu commited on
Commit
9a75dd9
·
verified ·
1 Parent(s): 301903b

Upload 14 files

Browse files
Files changed (14) hide show
  1. added_tokens.json +4 -0
  2. config.json +33 -0
  3. generation_config.json +7 -0
  4. merges.txt +0 -0
  5. openvino_detokenizer.bin +3 -0
  6. openvino_detokenizer.xml +223 -0
  7. openvino_model.bin +3 -0
  8. openvino_model.xml +0 -0
  9. openvino_tokenizer.bin +3 -0
  10. openvino_tokenizer.xml +965 -0
  11. special_tokens_map.json +70 -0
  12. tokenizer.json +0 -0
  13. tokenizer_config.json +374 -0
  14. vocab.json +0 -0
added_tokens.json ADDED
@@ -0,0 +1,4 @@
 
 
 
 
 
1
+ {
2
+ "<|im_end|>": 49152,
3
+ "<|im_start|>": 49153
4
+ }
config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_name_or_path": "/media/ecomm/c0889304-9e30-4f04-b290-c7db463872c6/Models/Pytorch/dolphincoder-starcoder2-15b",
3
+ "architectures": [
4
+ "Starcoder2ForCausalLM"
5
+ ],
6
+ "attention_dropout": 0.0,
7
+ "bos_token_id": 0,
8
+ "embedding_dropout": 0.0,
9
+ "eos_token_id": 0,
10
+ "hidden_act": "gelu_pytorch_tanh",
11
+ "hidden_size": 6144,
12
+ "initializer_range": 0.01275,
13
+ "intermediate_size": 24576,
14
+ "max_position_embeddings": 16384,
15
+ "mlp_type": "default",
16
+ "model_type": "starcoder2",
17
+ "norm_epsilon": 1e-05,
18
+ "norm_type": "layer_norm",
19
+ "num_attention_heads": 48,
20
+ "num_hidden_layers": 40,
21
+ "num_key_value_heads": 4,
22
+ "residual_dropout": 0.0,
23
+ "rms_norm_eps": 1e-05,
24
+ "rope_scaling": null,
25
+ "rope_theta": 100000,
26
+ "sliding_window": 4096,
27
+ "tie_word_embeddings": false,
28
+ "torch_dtype": "float16",
29
+ "transformers_version": "4.45.0.dev0",
30
+ "use_bias": true,
31
+ "use_cache": false,
32
+ "vocab_size": 49154
33
+ }
generation_config.json ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 50256,
4
+ "do_sample": true,
5
+ "eos_token_id": 50256,
6
+ "transformers_version": "4.45.0.dev0"
7
+ }
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
openvino_detokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:694b83ea2fa0659bb0127d0f7e2d824b96c9d6be5c25ebd3dfd7fb68b1794997
3
+ size 541550
openvino_detokenizer.xml ADDED
@@ -0,0 +1,223 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0"?>
2
+ <net name="detokenizer" version="11">
3
+ <layers>
4
+ <layer id="0" name="Parameter_302335" type="Parameter" version="opset1">
5
+ <data shape="?,?" element_type="i64" />
6
+ <output>
7
+ <port id="0" precision="I64" names="Parameter_302335">
8
+ <dim>-1</dim>
9
+ <dim>-1</dim>
10
+ </port>
11
+ </output>
12
+ </layer>
13
+ <layer id="1" name="Convert_302351" type="Convert" version="opset1">
14
+ <data destination_type="i32" />
15
+ <input>
16
+ <port id="0" precision="I64">
17
+ <dim>-1</dim>
18
+ <dim>-1</dim>
19
+ </port>
20
+ </input>
21
+ <output>
22
+ <port id="1" precision="I32">
23
+ <dim>-1</dim>
24
+ <dim>-1</dim>
25
+ </port>
26
+ </output>
27
+ </layer>
28
+ <layer id="2" name="Constant_302238" type="Const" version="opset1">
29
+ <data element_type="u8" shape="541501" offset="0" size="541501" />
30
+ <output>
31
+ <port id="0" precision="U8">
32
+ <dim>541501</dim>
33
+ </port>
34
+ </output>
35
+ </layer>
36
+ <layer id="3" name="StringTensorUnpack_302239" type="StringTensorUnpack" version="extension">
37
+ <data mode="begins_ends" />
38
+ <input>
39
+ <port id="0" precision="U8">
40
+ <dim>541501</dim>
41
+ </port>
42
+ </input>
43
+ <output>
44
+ <port id="1" precision="I32">
45
+ <dim>-1</dim>
46
+ </port>
47
+ <port id="2" precision="I32">
48
+ <dim>-1</dim>
49
+ </port>
50
+ <port id="3" precision="U8">
51
+ <dim>-1</dim>
52
+ </port>
53
+ </output>
54
+ </layer>
55
+ <layer id="4" name="VocabDecoder_302336" type="VocabDecoder" version="extension">
56
+ <data skip_tokens="0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 49152" />
57
+ <input>
58
+ <port id="0" precision="I32">
59
+ <dim>-1</dim>
60
+ <dim>-1</dim>
61
+ </port>
62
+ <port id="1" precision="I32">
63
+ <dim>-1</dim>
64
+ </port>
65
+ <port id="2" precision="I32">
66
+ <dim>-1</dim>
67
+ </port>
68
+ <port id="3" precision="U8">
69
+ <dim>-1</dim>
70
+ </port>
71
+ </input>
72
+ <output>
73
+ <port id="4" precision="I32">
74
+ <dim>-1</dim>
75
+ </port>
76
+ <port id="5" precision="I32">
77
+ <dim>-1</dim>
78
+ </port>
79
+ <port id="6" precision="I32">
80
+ <dim>-1</dim>
81
+ </port>
82
+ <port id="7" precision="I32">
83
+ <dim>-1</dim>
84
+ </port>
85
+ <port id="8" precision="U8">
86
+ <dim>-1</dim>
87
+ </port>
88
+ </output>
89
+ </layer>
90
+ <layer id="5" name="CharsToBytes_302337" type="CharsToBytes" version="extension">
91
+ <input>
92
+ <port id="0" precision="I32">
93
+ <dim>-1</dim>
94
+ </port>
95
+ <port id="1" precision="I32">
96
+ <dim>-1</dim>
97
+ </port>
98
+ <port id="2" precision="I32">
99
+ <dim>-1</dim>
100
+ </port>
101
+ <port id="3" precision="I32">
102
+ <dim>-1</dim>
103
+ </port>
104
+ <port id="4" precision="U8">
105
+ <dim>-1</dim>
106
+ </port>
107
+ </input>
108
+ <output>
109
+ <port id="5" precision="I32">
110
+ <dim>-1</dim>
111
+ </port>
112
+ <port id="6" precision="I32">
113
+ <dim>-1</dim>
114
+ </port>
115
+ <port id="7" precision="U8">
116
+ <dim>-1</dim>
117
+ </port>
118
+ </output>
119
+ </layer>
120
+ <layer id="6" name="Constant_302339" type="Const" version="opset1">
121
+ <data element_type="u8" shape="47" offset="541501" size="47" />
122
+ <output>
123
+ <port id="0" precision="U8">
124
+ <dim>47</dim>
125
+ </port>
126
+ </output>
127
+ </layer>
128
+ <layer id="7" name="Constant_302341" type="Const" version="opset1">
129
+ <data element_type="u8" shape="2" offset="541548" size="2" />
130
+ <output>
131
+ <port id="0" precision="U8">
132
+ <dim>2</dim>
133
+ </port>
134
+ </output>
135
+ </layer>
136
+ <layer id="8" name="RegexNormalization_302342" type="RegexNormalization" version="extension">
137
+ <data global_replace="true" />
138
+ <input>
139
+ <port id="0" precision="I32">
140
+ <dim>-1</dim>
141
+ </port>
142
+ <port id="1" precision="I32">
143
+ <dim>-1</dim>
144
+ </port>
145
+ <port id="2" precision="U8">
146
+ <dim>-1</dim>
147
+ </port>
148
+ <port id="3" precision="U8">
149
+ <dim>47</dim>
150
+ </port>
151
+ <port id="4" precision="U8">
152
+ <dim>2</dim>
153
+ </port>
154
+ </input>
155
+ <output>
156
+ <port id="5" precision="I32">
157
+ <dim>-1</dim>
158
+ </port>
159
+ <port id="6" precision="I32">
160
+ <dim>-1</dim>
161
+ </port>
162
+ <port id="7" precision="U8">
163
+ <dim>-1</dim>
164
+ </port>
165
+ </output>
166
+ </layer>
167
+ <layer id="9" name="StringTensorPack_302343" type="StringTensorPack" version="extension">
168
+ <data mode="begins_ends" />
169
+ <input>
170
+ <port id="0" precision="I32">
171
+ <dim>-1</dim>
172
+ </port>
173
+ <port id="1" precision="I32">
174
+ <dim>-1</dim>
175
+ </port>
176
+ <port id="2" precision="U8">
177
+ <dim>-1</dim>
178
+ </port>
179
+ </input>
180
+ <output>
181
+ <port id="3" precision="STRING" names="string_output">
182
+ <dim>-1</dim>
183
+ </port>
184
+ </output>
185
+ </layer>
186
+ <layer id="10" name="Result_302344" type="Result" version="opset1">
187
+ <input>
188
+ <port id="0" precision="STRING">
189
+ <dim>-1</dim>
190
+ </port>
191
+ </input>
192
+ </layer>
193
+ </layers>
194
+ <edges>
195
+ <edge from-layer="0" from-port="0" to-layer="1" to-port="0" />
196
+ <edge from-layer="1" from-port="1" to-layer="4" to-port="0" />
197
+ <edge from-layer="2" from-port="0" to-layer="3" to-port="0" />
198
+ <edge from-layer="3" from-port="1" to-layer="4" to-port="1" />
199
+ <edge from-layer="3" from-port="2" to-layer="4" to-port="2" />
200
+ <edge from-layer="3" from-port="3" to-layer="4" to-port="3" />
201
+ <edge from-layer="4" from-port="8" to-layer="5" to-port="4" />
202
+ <edge from-layer="4" from-port="7" to-layer="5" to-port="3" />
203
+ <edge from-layer="4" from-port="6" to-layer="5" to-port="2" />
204
+ <edge from-layer="4" from-port="5" to-layer="5" to-port="1" />
205
+ <edge from-layer="4" from-port="4" to-layer="5" to-port="0" />
206
+ <edge from-layer="5" from-port="5" to-layer="8" to-port="0" />
207
+ <edge from-layer="5" from-port="6" to-layer="8" to-port="1" />
208
+ <edge from-layer="5" from-port="7" to-layer="8" to-port="2" />
209
+ <edge from-layer="6" from-port="0" to-layer="8" to-port="3" />
210
+ <edge from-layer="7" from-port="0" to-layer="8" to-port="4" />
211
+ <edge from-layer="8" from-port="5" to-layer="9" to-port="0" />
212
+ <edge from-layer="8" from-port="6" to-layer="9" to-port="1" />
213
+ <edge from-layer="8" from-port="7" to-layer="9" to-port="2" />
214
+ <edge from-layer="9" from-port="3" to-layer="10" to-port="0" />
215
+ </edges>
216
+ <rt_info>
217
+ <bos_token_id value="0" />
218
+ <chat_template value="{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'&lt;|im_start|>system&#10;' + system_message + '&lt;|im_end|>&#10;'}}{% endif %}{{'&lt;|im_start|>' + message['role'] + '&#10;' + message['content'] + '&lt;|im_end|>' + '&#10;'}}{% endfor %}{% if add_generation_prompt %}{{ '&lt;|im_start|>assistant&#10;' }}{% endif %}" />
219
+ <eos_token_id value="49152" />
220
+ <original_tokenizer_class value="&lt;class 'transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast'>" />
221
+ <pad_token_id value="0" />
222
+ </rt_info>
223
+ </net>
openvino_model.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:9e02fda21c471ddfb857aa7fd92e6a94aaabf3f461f9a6d4f1ba3bed10a33876
3
+ size 31915827614
openvino_model.xml ADDED
The diff for this file is too large to render. See raw diff
 
openvino_tokenizer.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d77fb0675443b8851af06f1329a8beb819885520cf90d279ef62ec14dadd0881
3
+ size 1132109
openvino_tokenizer.xml ADDED
@@ -0,0 +1,965 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ <?xml version="1.0"?>
2
+ <net name="tokenizer" version="11">
3
+ <layers>
4
+ <layer id="0" name="Parameter_302145" type="Parameter" version="opset1">
5
+ <data shape="?" element_type="string" />
6
+ <output>
7
+ <port id="0" precision="STRING" names="Parameter_302145">
8
+ <dim>-1</dim>
9
+ </port>
10
+ </output>
11
+ </layer>
12
+ <layer id="1" name="Constant_302151" type="Const" version="opset1">
13
+ <data element_type="i64" shape="" offset="0" size="8" />
14
+ <output>
15
+ <port id="0" precision="I64" />
16
+ </output>
17
+ </layer>
18
+ <layer id="2" name="StringTensorUnpack_302146" type="StringTensorUnpack" version="extension">
19
+ <data mode="begins_ends" />
20
+ <input>
21
+ <port id="0" precision="STRING">
22
+ <dim>-1</dim>
23
+ </port>
24
+ </input>
25
+ <output>
26
+ <port id="1" precision="I32">
27
+ <dim>-1</dim>
28
+ </port>
29
+ <port id="2" precision="I32">
30
+ <dim>-1</dim>
31
+ </port>
32
+ <port id="3" precision="U8">
33
+ <dim>-1</dim>
34
+ </port>
35
+ </output>
36
+ </layer>
37
+ <layer id="3" name="ShapeOf_302147" type="ShapeOf" version="opset3">
38
+ <data output_type="i64" />
39
+ <input>
40
+ <port id="0" precision="I32">
41
+ <dim>-1</dim>
42
+ </port>
43
+ </input>
44
+ <output>
45
+ <port id="1" precision="I64">
46
+ <dim>1</dim>
47
+ </port>
48
+ </output>
49
+ </layer>
50
+ <layer id="4" name="Constant_302148" type="Const" version="opset1">
51
+ <data element_type="i64" shape="" offset="0" size="8" />
52
+ <output>
53
+ <port id="0" precision="I64" />
54
+ </output>
55
+ </layer>
56
+ <layer id="5" name="Constant_302149" type="Const" version="opset1">
57
+ <data element_type="i64" shape="" offset="0" size="8" />
58
+ <output>
59
+ <port id="0" precision="I64" />
60
+ </output>
61
+ </layer>
62
+ <layer id="6" name="Gather_302150" type="Gather" version="opset8">
63
+ <data batch_dims="0" />
64
+ <input>
65
+ <port id="0" precision="I64">
66
+ <dim>1</dim>
67
+ </port>
68
+ <port id="1" precision="I64" />
69
+ <port id="2" precision="I64" />
70
+ </input>
71
+ <output>
72
+ <port id="3" precision="I64" />
73
+ </output>
74
+ </layer>
75
+ <layer id="7" name="Constant_302152" type="Const" version="opset1">
76
+ <data element_type="i64" shape="" offset="8" size="8" />
77
+ <output>
78
+ <port id="0" precision="I64" />
79
+ </output>
80
+ </layer>
81
+ <layer id="8" name="Range_302153" type="Range" version="opset4">
82
+ <data output_type="i32" />
83
+ <input>
84
+ <port id="0" precision="I64" />
85
+ <port id="1" precision="I64" />
86
+ <port id="2" precision="I64" />
87
+ </input>
88
+ <output>
89
+ <port id="3" precision="I32">
90
+ <dim>-1</dim>
91
+ </port>
92
+ </output>
93
+ </layer>
94
+ <layer id="9" name="Constant_302155" type="Const" version="opset1">
95
+ <data element_type="i64" shape="" offset="8" size="8" />
96
+ <output>
97
+ <port id="0" precision="I64" />
98
+ </output>
99
+ </layer>
100
+ <layer id="10" name="Constant_302156" type="Const" version="opset1">
101
+ <data element_type="i64" shape="" offset="8" size="8" />
102
+ <output>
103
+ <port id="0" precision="I64" />
104
+ </output>
105
+ </layer>
106
+ <layer id="11" name="Add_302157" type="Add" version="opset1">
107
+ <data auto_broadcast="numpy" />
108
+ <input>
109
+ <port id="0" precision="I64" />
110
+ <port id="1" precision="I64" />
111
+ </input>
112
+ <output>
113
+ <port id="2" precision="I64" />
114
+ </output>
115
+ </layer>
116
+ <layer id="12" name="Constant_302158" type="Const" version="opset1">
117
+ <data element_type="i64" shape="" offset="8" size="8" />
118
+ <output>
119
+ <port id="0" precision="I64" />
120
+ </output>
121
+ </layer>
122
+ <layer id="13" name="Range_302159" type="Range" version="opset4">
123
+ <data output_type="i32" />
124
+ <input>
125
+ <port id="0" precision="I64" />
126
+ <port id="1" precision="I64" />
127
+ <port id="2" precision="I64" />
128
+ </input>
129
+ <output>
130
+ <port id="3" precision="I32">
131
+ <dim>-1</dim>
132
+ </port>
133
+ </output>
134
+ </layer>
135
+ <layer id="14" name="Constant_302222" type="Const" version="opset1">
136
+ <data element_type="u8" shape="647" offset="16" size="647" />
137
+ <output>
138
+ <port id="0" precision="U8">
139
+ <dim>647</dim>
140
+ </port>
141
+ </output>
142
+ </layer>
143
+ <layer id="15" name="RegexSplit_302223" type="RegexSplit" version="extension">
144
+ <data behaviour="isolate" invert="false" max_splits="-1" />
145
+ <input>
146
+ <port id="0" precision="I32">
147
+ <dim>-1</dim>
148
+ </port>
149
+ <port id="1" precision="I32">
150
+ <dim>-1</dim>
151
+ </port>
152
+ <port id="2" precision="I32">
153
+ <dim>-1</dim>
154
+ </port>
155
+ <port id="3" precision="I32">
156
+ <dim>-1</dim>
157
+ </port>
158
+ <port id="4" precision="U8">
159
+ <dim>-1</dim>
160
+ </port>
161
+ <port id="5" precision="U8">
162
+ <dim>647</dim>
163
+ </port>
164
+ </input>
165
+ <output>
166
+ <port id="6" precision="I32">
167
+ <dim>-1</dim>
168
+ </port>
169
+ <port id="7" precision="I32">
170
+ <dim>-1</dim>
171
+ </port>
172
+ <port id="8" precision="I32">
173
+ <dim>-1</dim>
174
+ </port>
175
+ <port id="9" precision="I32">
176
+ <dim>-1</dim>
177
+ </port>
178
+ <port id="10" precision="U8">
179
+ <dim>-1</dim>
180
+ </port>
181
+ </output>
182
+ </layer>
183
+ <layer id="16" name="Constant_302228" type="Const" version="opset1">
184
+ <data element_type="u8" shape="20" offset="663" size="20" />
185
+ <output>
186
+ <port id="0" precision="U8">
187
+ <dim>20</dim>
188
+ </port>
189
+ </output>
190
+ </layer>
191
+ <layer id="17" name="Constant_302225" type="Const" version="opset1">
192
+ <data element_type="u8" shape="691" offset="683" size="691" />
193
+ <output>
194
+ <port id="0" precision="U8">
195
+ <dim>691</dim>
196
+ </port>
197
+ </output>
198
+ </layer>
199
+ <layer id="18" name="StringTensorUnpack_302226" type="StringTensorUnpack" version="extension">
200
+ <data mode="begins_ends" />
201
+ <input>
202
+ <port id="0" precision="U8">
203
+ <dim>691</dim>
204
+ </port>
205
+ </input>
206
+ <output>
207
+ <port id="1" precision="I32">
208
+ <dim>-1</dim>
209
+ </port>
210
+ <port id="2" precision="I32">
211
+ <dim>-1</dim>
212
+ </port>
213
+ <port id="3" precision="U8">
214
+ <dim>-1</dim>
215
+ </port>
216
+ </output>
217
+ </layer>
218
+ <layer id="19" name="RegexSplit_302229" type="RegexSplit" version="extension">
219
+ <data behaviour="isolate" invert="false" max_splits="-1" />
220
+ <input>
221
+ <port id="0" precision="I32">
222
+ <dim>-1</dim>
223
+ </port>
224
+ <port id="1" precision="I32">
225
+ <dim>-1</dim>
226
+ </port>
227
+ <port id="2" precision="I32">
228
+ <dim>-1</dim>
229
+ </port>
230
+ <port id="3" precision="I32">
231
+ <dim>-1</dim>
232
+ </port>
233
+ <port id="4" precision="U8">
234
+ <dim>-1</dim>
235
+ </port>
236
+ <port id="5" precision="U8">
237
+ <dim>20</dim>
238
+ </port>
239
+ <port id="6" precision="I32">
240
+ <dim>-1</dim>
241
+ </port>
242
+ <port id="7" precision="I32">
243
+ <dim>-1</dim>
244
+ </port>
245
+ <port id="8" precision="U8">
246
+ <dim>-1</dim>
247
+ </port>
248
+ </input>
249
+ <output>
250
+ <port id="9" precision="I32">
251
+ <dim>-1</dim>
252
+ </port>
253
+ <port id="10" precision="I32">
254
+ <dim>-1</dim>
255
+ </port>
256
+ <port id="11" precision="I32">
257
+ <dim>-1</dim>
258
+ </port>
259
+ <port id="12" precision="I32">
260
+ <dim>-1</dim>
261
+ </port>
262
+ <port id="13" precision="U8">
263
+ <dim>-1</dim>
264
+ </port>
265
+ </output>
266
+ </layer>
267
+ <layer id="20" name="Constant_302234" type="Const" version="opset1">
268
+ <data element_type="u8" shape="64" offset="1374" size="64" />
269
+ <output>
270
+ <port id="0" precision="U8">
271
+ <dim>64</dim>
272
+ </port>
273
+ </output>
274
+ </layer>
275
+ <layer id="21" name="Constant_302231" type="Const" version="opset1">
276
+ <data element_type="u8" shape="691" offset="683" size="691" />
277
+ <output>
278
+ <port id="0" precision="U8">
279
+ <dim>691</dim>
280
+ </port>
281
+ </output>
282
+ </layer>
283
+ <layer id="22" name="StringTensorUnpack_302232" type="StringTensorUnpack" version="extension">
284
+ <data mode="begins_ends" />
285
+ <input>
286
+ <port id="0" precision="U8">
287
+ <dim>691</dim>
288
+ </port>
289
+ </input>
290
+ <output>
291
+ <port id="1" precision="I32">
292
+ <dim>-1</dim>
293
+ </port>
294
+ <port id="2" precision="I32">
295
+ <dim>-1</dim>
296
+ </port>
297
+ <port id="3" precision="U8">
298
+ <dim>-1</dim>
299
+ </port>
300
+ </output>
301
+ </layer>
302
+ <layer id="23" name="RegexSplit_302235" type="RegexSplit" version="extension">
303
+ <data behaviour="isolate" invert="false" max_splits="-1" />
304
+ <input>
305
+ <port id="0" precision="I32">
306
+ <dim>-1</dim>
307
+ </port>
308
+ <port id="1" precision="I32">
309
+ <dim>-1</dim>
310
+ </port>
311
+ <port id="2" precision="I32">
312
+ <dim>-1</dim>
313
+ </port>
314
+ <port id="3" precision="I32">
315
+ <dim>-1</dim>
316
+ </port>
317
+ <port id="4" precision="U8">
318
+ <dim>-1</dim>
319
+ </port>
320
+ <port id="5" precision="U8">
321
+ <dim>64</dim>
322
+ </port>
323
+ <port id="6" precision="I32">
324
+ <dim>-1</dim>
325
+ </port>
326
+ <port id="7" precision="I32">
327
+ <dim>-1</dim>
328
+ </port>
329
+ <port id="8" precision="U8">
330
+ <dim>-1</dim>
331
+ </port>
332
+ </input>
333
+ <output>
334
+ <port id="9" precision="I32">
335
+ <dim>-1</dim>
336
+ </port>
337
+ <port id="10" precision="I32">
338
+ <dim>-1</dim>
339
+ </port>
340
+ <port id="11" precision="I32">
341
+ <dim>-1</dim>
342
+ </port>
343
+ <port id="12" precision="I32">
344
+ <dim>-1</dim>
345
+ </port>
346
+ <port id="13" precision="U8">
347
+ <dim>-1</dim>
348
+ </port>
349
+ </output>
350
+ </layer>
351
+ <layer id="24" name="BytesToChars_302236" type="BytesToChars" version="extension">
352
+ <input>
353
+ <port id="0" precision="I32">
354
+ <dim>-1</dim>
355
+ </port>
356
+ <port id="1" precision="I32">
357
+ <dim>-1</dim>
358
+ </port>
359
+ <port id="2" precision="I32">
360
+ <dim>-1</dim>
361
+ </port>
362
+ <port id="3" precision="I32">
363
+ <dim>-1</dim>
364
+ </port>
365
+ <port id="4" precision="U8">
366
+ <dim>-1</dim>
367
+ </port>
368
+ </input>
369
+ <output>
370
+ <port id="5" precision="I32">
371
+ <dim>-1</dim>
372
+ </port>
373
+ <port id="6" precision="I32">
374
+ <dim>-1</dim>
375
+ </port>
376
+ <port id="7" precision="I32">
377
+ <dim>-1</dim>
378
+ </port>
379
+ <port id="8" precision="I32">
380
+ <dim>-1</dim>
381
+ </port>
382
+ <port id="9" precision="U8">
383
+ <dim>-1</dim>
384
+ </port>
385
+ </output>
386
+ </layer>
387
+ <layer id="25" name="Constant_302238" type="Const" version="opset1">
388
+ <data element_type="u8" shape="541501" offset="1438" size="541501" />
389
+ <output>
390
+ <port id="0" precision="U8">
391
+ <dim>541501</dim>
392
+ </port>
393
+ </output>
394
+ </layer>
395
+ <layer id="26" name="StringTensorUnpack_302239" type="StringTensorUnpack" version="extension">
396
+ <data mode="begins_ends" />
397
+ <input>
398
+ <port id="0" precision="U8">
399
+ <dim>541501</dim>
400
+ </port>
401
+ </input>
402
+ <output>
403
+ <port id="1" precision="I32">
404
+ <dim>-1</dim>
405
+ </port>
406
+ <port id="2" precision="I32">
407
+ <dim>-1</dim>
408
+ </port>
409
+ <port id="3" precision="U8">
410
+ <dim>-1</dim>
411
+ </port>
412
+ </output>
413
+ </layer>
414
+ <layer id="27" name="Constant_302319" type="Const" version="opset1">
415
+ <data element_type="u8" shape="588315" offset="542939" size="588315" />
416
+ <output>
417
+ <port id="0" precision="U8">
418
+ <dim>588315</dim>
419
+ </port>
420
+ </output>
421
+ </layer>
422
+ <layer id="28" name="StringTensorUnpack_302320" type="StringTensorUnpack" version="extension">
423
+ <data mode="begins_ends" />
424
+ <input>
425
+ <port id="0" precision="U8">
426
+ <dim>588315</dim>
427
+ </port>
428
+ </input>
429
+ <output>
430
+ <port id="1" precision="I32">
431
+ <dim>-1</dim>
432
+ </port>
433
+ <port id="2" precision="I32">
434
+ <dim>-1</dim>
435
+ </port>
436
+ <port id="3" precision="U8">
437
+ <dim>-1</dim>
438
+ </port>
439
+ </output>
440
+ </layer>
441
+ <layer id="29" name="Constant_302247" type="Const" version="opset1">
442
+ <data element_type="i64" shape="" offset="0" size="8" />
443
+ <output>
444
+ <port id="0" precision="I64" />
445
+ </output>
446
+ </layer>
447
+ <layer id="30" name="Constant_302241" type="Const" version="opset1">
448
+ <data element_type="u8" shape="691" offset="1131254" size="691" />
449
+ <output>
450
+ <port id="0" precision="U8">
451
+ <dim>691</dim>
452
+ </port>
453
+ </output>
454
+ </layer>
455
+ <layer id="31" name="StringTensorUnpack_302242" type="StringTensorUnpack" version="extension">
456
+ <data mode="begins_ends" />
457
+ <input>
458
+ <port id="0" precision="U8">
459
+ <dim>691</dim>
460
+ </port>
461
+ </input>
462
+ <output>
463
+ <port id="1" precision="I32">
464
+ <dim>-1</dim>
465
+ </port>
466
+ <port id="2" precision="I32">
467
+ <dim>-1</dim>
468
+ </port>
469
+ <port id="3" precision="U8">
470
+ <dim>-1</dim>
471
+ </port>
472
+ </output>
473
+ </layer>
474
+ <layer id="32" name="ShapeOf_302243" type="ShapeOf" version="opset3">
475
+ <data output_type="i64" />
476
+ <input>
477
+ <port id="0" precision="I32">
478
+ <dim>-1</dim>
479
+ </port>
480
+ </input>
481
+ <output>
482
+ <port id="1" precision="I64">
483
+ <dim>1</dim>
484
+ </port>
485
+ </output>
486
+ </layer>
487
+ <layer id="33" name="Constant_302244" type="Const" version="opset1">
488
+ <data element_type="i64" shape="" offset="0" size="8" />
489
+ <output>
490
+ <port id="0" precision="I64" />
491
+ </output>
492
+ </layer>
493
+ <layer id="34" name="Constant_302245" type="Const" version="opset1">
494
+ <data element_type="i64" shape="" offset="0" size="8" />
495
+ <output>
496
+ <port id="0" precision="I64" />
497
+ </output>
498
+ </layer>
499
+ <layer id="35" name="Gather_302246" type="Gather" version="opset8">
500
+ <data batch_dims="0" />
501
+ <input>
502
+ <port id="0" precision="I64">
503
+ <dim>1</dim>
504
+ </port>
505
+ <port id="1" precision="I64" />
506
+ <port id="2" precision="I64" />
507
+ </input>
508
+ <output>
509
+ <port id="3" precision="I64" />
510
+ </output>
511
+ </layer>
512
+ <layer id="36" name="Constant_302248" type="Const" version="opset1">
513
+ <data element_type="i64" shape="" offset="8" size="8" />
514
+ <output>
515
+ <port id="0" precision="I64" />
516
+ </output>
517
+ </layer>
518
+ <layer id="37" name="Range_302249" type="Range" version="opset4">
519
+ <data output_type="i32" />
520
+ <input>
521
+ <port id="0" precision="I64" />
522
+ <port id="1" precision="I64" />
523
+ <port id="2" precision="I64" />
524
+ </input>
525
+ <output>
526
+ <port id="3" precision="I32">
527
+ <dim>-1</dim>
528
+ </port>
529
+ </output>
530
+ </layer>
531
+ <layer id="38" name="Constant_302251" type="Const" version="opset1">
532
+ <data element_type="i64" shape="" offset="8" size="8" />
533
+ <output>
534
+ <port id="0" precision="I64" />
535
+ </output>
536
+ </layer>
537
+ <layer id="39" name="Constant_302252" type="Const" version="opset1">
538
+ <data element_type="i64" shape="" offset="8" size="8" />
539
+ <output>
540
+ <port id="0" precision="I64" />
541
+ </output>
542
+ </layer>
543
+ <layer id="40" name="Add_302253" type="Add" version="opset1">
544
+ <data auto_broadcast="numpy" />
545
+ <input>
546
+ <port id="0" precision="I64" />
547
+ <port id="1" precision="I64" />
548
+ </input>
549
+ <output>
550
+ <port id="2" precision="I64" />
551
+ </output>
552
+ </layer>
553
+ <layer id="41" name="Constant_302254" type="Const" version="opset1">
554
+ <data element_type="i64" shape="" offset="8" size="8" />
555
+ <output>
556
+ <port id="0" precision="I64" />
557
+ </output>
558
+ </layer>
559
+ <layer id="42" name="Range_302255" type="Range" version="opset4">
560
+ <data output_type="i32" />
561
+ <input>
562
+ <port id="0" precision="I64" />
563
+ <port id="1" precision="I64" />
564
+ <port id="2" precision="I64" />
565
+ </input>
566
+ <output>
567
+ <port id="3" precision="I32">
568
+ <dim>-1</dim>
569
+ </port>
570
+ </output>
571
+ </layer>
572
+ <layer id="43" name="BytesToChars_302317" type="BytesToChars" version="extension">
573
+ <input>
574
+ <port id="0" precision="I32">
575
+ <dim>-1</dim>
576
+ </port>
577
+ <port id="1" precision="I32">
578
+ <dim>-1</dim>
579
+ </port>
580
+ <port id="2" precision="I32">
581
+ <dim>-1</dim>
582
+ </port>
583
+ <port id="3" precision="I32">
584
+ <dim>-1</dim>
585
+ </port>
586
+ <port id="4" precision="U8">
587
+ <dim>-1</dim>
588
+ </port>
589
+ </input>
590
+ <output>
591
+ <port id="5" precision="I32">
592
+ <dim>-1</dim>
593
+ </port>
594
+ <port id="6" precision="I32">
595
+ <dim>-1</dim>
596
+ </port>
597
+ <port id="7" precision="I32">
598
+ <dim>-1</dim>
599
+ </port>
600
+ <port id="8" precision="I32">
601
+ <dim>-1</dim>
602
+ </port>
603
+ <port id="9" precision="U8">
604
+ <dim>-1</dim>
605
+ </port>
606
+ </output>
607
+ </layer>
608
+ <layer id="44" name="Constant_302321" type="Const" version="opset1">
609
+ <data element_type="i32" shape="39" offset="1131945" size="156" />
610
+ <output>
611
+ <port id="0" precision="I32">
612
+ <dim>39</dim>
613
+ </port>
614
+ </output>
615
+ </layer>
616
+ <layer id="45" name="BPETokenizer_302322" type="BPETokenizer" version="extension">
617
+ <data unk_token="" fuse_unk="false" suffix_indicator="" end_suffix="" byte_fallback="false" cache_capacity="20000" />
618
+ <input>
619
+ <port id="0" precision="I32">
620
+ <dim>-1</dim>
621
+ </port>
622
+ <port id="1" precision="I32">
623
+ <dim>-1</dim>
624
+ </port>
625
+ <port id="2" precision="I32">
626
+ <dim>-1</dim>
627
+ </port>
628
+ <port id="3" precision="I32">
629
+ <dim>-1</dim>
630
+ </port>
631
+ <port id="4" precision="U8">
632
+ <dim>-1</dim>
633
+ </port>
634
+ <port id="5" precision="I32">
635
+ <dim>-1</dim>
636
+ </port>
637
+ <port id="6" precision="I32">
638
+ <dim>-1</dim>
639
+ </port>
640
+ <port id="7" precision="U8">
641
+ <dim>-1</dim>
642
+ </port>
643
+ <port id="8" precision="I32">
644
+ <dim>-1</dim>
645
+ </port>
646
+ <port id="9" precision="I32">
647
+ <dim>-1</dim>
648
+ </port>
649
+ <port id="10" precision="U8">
650
+ <dim>-1</dim>
651
+ </port>
652
+ <port id="11" precision="I32">
653
+ <dim>-1</dim>
654
+ </port>
655
+ <port id="12" precision="I32">
656
+ <dim>-1</dim>
657
+ </port>
658
+ <port id="13" precision="U8">
659
+ <dim>-1</dim>
660
+ </port>
661
+ <port id="14" precision="I32">
662
+ <dim>39</dim>
663
+ </port>
664
+ </input>
665
+ <output>
666
+ <port id="15" precision="I32">
667
+ <dim>-1</dim>
668
+ </port>
669
+ <port id="16" precision="I32">
670
+ <dim>-1</dim>
671
+ </port>
672
+ <port id="17" precision="I32">
673
+ <dim>-1</dim>
674
+ </port>
675
+ </output>
676
+ </layer>
677
+ <layer id="46" name="Subtract_302323" type="Subtract" version="opset1">
678
+ <data auto_broadcast="numpy" />
679
+ <input>
680
+ <port id="0" precision="I32">
681
+ <dim>-1</dim>
682
+ </port>
683
+ <port id="1" precision="I32">
684
+ <dim>-1</dim>
685
+ </port>
686
+ </input>
687
+ <output>
688
+ <port id="2" precision="I32">
689
+ <dim>-1</dim>
690
+ </port>
691
+ </output>
692
+ </layer>
693
+ <layer id="47" name="Constant_302324" type="Const" version="opset1">
694
+ <data element_type="i32" shape="" offset="1132101" size="4" />
695
+ <output>
696
+ <port id="0" precision="I32" />
697
+ </output>
698
+ </layer>
699
+ <layer id="48" name="Minimum_302325" type="Minimum" version="opset1">
700
+ <data auto_broadcast="numpy" />
701
+ <input>
702
+ <port id="0" precision="I32">
703
+ <dim>-1</dim>
704
+ </port>
705
+ <port id="1" precision="I32" />
706
+ </input>
707
+ <output>
708
+ <port id="2" precision="I32">
709
+ <dim>-1</dim>
710
+ </port>
711
+ </output>
712
+ </layer>
713
+ <layer id="49" name="Subtract_302326" type="Subtract" version="opset1">
714
+ <data auto_broadcast="numpy" />
715
+ <input>
716
+ <port id="0" precision="I32">
717
+ <dim>-1</dim>
718
+ </port>
719
+ <port id="1" precision="I32">
720
+ <dim>-1</dim>
721
+ </port>
722
+ </input>
723
+ <output>
724
+ <port id="2" precision="I32">
725
+ <dim>-1</dim>
726
+ </port>
727
+ </output>
728
+ </layer>
729
+ <layer id="50" name="Subtract_302327" type="Subtract" version="opset1">
730
+ <data auto_broadcast="numpy" />
731
+ <input>
732
+ <port id="0" precision="I32">
733
+ <dim>-1</dim>
734
+ </port>
735
+ <port id="1" precision="I32">
736
+ <dim>-1</dim>
737
+ </port>
738
+ </input>
739
+ <output>
740
+ <port id="2" precision="I32">
741
+ <dim>-1</dim>
742
+ </port>
743
+ </output>
744
+ </layer>
745
+ <layer id="51" name="Constant_302328" type="Const" version="opset1">
746
+ <data element_type="i32" shape="" offset="1132105" size="4" />
747
+ <output>
748
+ <port id="0" precision="I32" />
749
+ </output>
750
+ </layer>
751
+ <layer id="52" name="ReduceMax_302329" type="ReduceMax" version="opset1">
752
+ <data keep_dims="false" />
753
+ <input>
754
+ <port id="0" precision="I32">
755
+ <dim>-1</dim>
756
+ </port>
757
+ <port id="1" precision="I32" />
758
+ </input>
759
+ <output>
760
+ <port id="2" precision="I32" />
761
+ </output>
762
+ </layer>
763
+ <layer id="53" name="Constant_302330" type="Const" version="opset1">
764
+ <data element_type="i32" shape="" offset="1132105" size="4" />
765
+ <output>
766
+ <port id="0" precision="I32" />
767
+ </output>
768
+ </layer>
769
+ <layer id="54" name="RaggedToDense_302331" type="RaggedToDense" version="extension">
770
+ <data pad_right="false" />
771
+ <input>
772
+ <port id="0" precision="I32">
773
+ <dim>-1</dim>
774
+ </port>
775
+ <port id="1" precision="I32">
776
+ <dim>-1</dim>
777
+ </port>
778
+ <port id="2" precision="I32">
779
+ <dim>-1</dim>
780
+ </port>
781
+ <port id="3" precision="I32" />
782
+ <port id="4" precision="I32" />
783
+ </input>
784
+ <output>
785
+ <port id="5" precision="I32">
786
+ <dim>-1</dim>
787
+ <dim>-1</dim>
788
+ </port>
789
+ <port id="6" precision="BOOL">
790
+ <dim>-1</dim>
791
+ <dim>-1</dim>
792
+ </port>
793
+ </output>
794
+ </layer>
795
+ <layer id="55" name="Convert_302332" type="Convert" version="opset1">
796
+ <data destination_type="i32" />
797
+ <input>
798
+ <port id="0" precision="BOOL">
799
+ <dim>-1</dim>
800
+ <dim>-1</dim>
801
+ </port>
802
+ </input>
803
+ <output>
804
+ <port id="1" precision="I32">
805
+ <dim>-1</dim>
806
+ <dim>-1</dim>
807
+ </port>
808
+ </output>
809
+ </layer>
810
+ <layer id="56" name="Convert_302332" type="Convert" version="opset1">
811
+ <data destination_type="i64" />
812
+ <input>
813
+ <port id="0" precision="I32">
814
+ <dim>-1</dim>
815
+ <dim>-1</dim>
816
+ </port>
817
+ </input>
818
+ <output>
819
+ <port id="1" precision="I64" names="attention_mask">
820
+ <dim>-1</dim>
821
+ <dim>-1</dim>
822
+ </port>
823
+ </output>
824
+ </layer>
825
+ <layer id="58" name="RaggedToDense_302331.0" type="Convert" version="opset1">
826
+ <data destination_type="i64" />
827
+ <input>
828
+ <port id="0" precision="I32">
829
+ <dim>-1</dim>
830
+ <dim>-1</dim>
831
+ </port>
832
+ </input>
833
+ <output>
834
+ <port id="1" precision="I64" names="input_ids">
835
+ <dim>-1</dim>
836
+ <dim>-1</dim>
837
+ </port>
838
+ </output>
839
+ </layer>
840
+ <layer id="59" name="Result_302333" type="Result" version="opset1">
841
+ <input>
842
+ <port id="0" precision="I64">
843
+ <dim>-1</dim>
844
+ <dim>-1</dim>
845
+ </port>
846
+ </input>
847
+ </layer>
848
+ <layer id="57" name="Result_302334" type="Result" version="opset1">
849
+ <input>
850
+ <port id="0" precision="I64">
851
+ <dim>-1</dim>
852
+ <dim>-1</dim>
853
+ </port>
854
+ </input>
855
+ </layer>
856
+ </layers>
857
+ <edges>
858
+ <edge from-layer="0" from-port="0" to-layer="2" to-port="0" />
859
+ <edge from-layer="1" from-port="0" to-layer="8" to-port="0" />
860
+ <edge from-layer="2" from-port="1" to-layer="3" to-port="0" />
861
+ <edge from-layer="2" from-port="3" to-layer="15" to-port="4" />
862
+ <edge from-layer="2" from-port="2" to-layer="15" to-port="3" />
863
+ <edge from-layer="2" from-port="1" to-layer="15" to-port="2" />
864
+ <edge from-layer="3" from-port="1" to-layer="6" to-port="0" />
865
+ <edge from-layer="4" from-port="0" to-layer="6" to-port="1" />
866
+ <edge from-layer="5" from-port="0" to-layer="6" to-port="2" />
867
+ <edge from-layer="6" from-port="3" to-layer="8" to-port="1" />
868
+ <edge from-layer="6" from-port="3" to-layer="11" to-port="0" />
869
+ <edge from-layer="7" from-port="0" to-layer="8" to-port="2" />
870
+ <edge from-layer="8" from-port="3" to-layer="15" to-port="0" />
871
+ <edge from-layer="9" from-port="0" to-layer="13" to-port="0" />
872
+ <edge from-layer="10" from-port="0" to-layer="11" to-port="1" />
873
+ <edge from-layer="11" from-port="2" to-layer="13" to-port="1" />
874
+ <edge from-layer="12" from-port="0" to-layer="13" to-port="2" />
875
+ <edge from-layer="13" from-port="3" to-layer="15" to-port="1" />
876
+ <edge from-layer="14" from-port="0" to-layer="15" to-port="5" />
877
+ <edge from-layer="15" from-port="6" to-layer="19" to-port="0" />
878
+ <edge from-layer="15" from-port="7" to-layer="19" to-port="1" />
879
+ <edge from-layer="15" from-port="8" to-layer="19" to-port="2" />
880
+ <edge from-layer="15" from-port="9" to-layer="19" to-port="3" />
881
+ <edge from-layer="15" from-port="10" to-layer="19" to-port="4" />
882
+ <edge from-layer="16" from-port="0" to-layer="19" to-port="5" />
883
+ <edge from-layer="17" from-port="0" to-layer="18" to-port="0" />
884
+ <edge from-layer="18" from-port="1" to-layer="19" to-port="6" />
885
+ <edge from-layer="18" from-port="2" to-layer="19" to-port="7" />
886
+ <edge from-layer="18" from-port="3" to-layer="19" to-port="8" />
887
+ <edge from-layer="19" from-port="11" to-layer="23" to-port="2" />
888
+ <edge from-layer="19" from-port="13" to-layer="23" to-port="4" />
889
+ <edge from-layer="19" from-port="12" to-layer="23" to-port="3" />
890
+ <edge from-layer="19" from-port="10" to-layer="23" to-port="1" />
891
+ <edge from-layer="19" from-port="9" to-layer="23" to-port="0" />
892
+ <edge from-layer="20" from-port="0" to-layer="23" to-port="5" />
893
+ <edge from-layer="21" from-port="0" to-layer="22" to-port="0" />
894
+ <edge from-layer="22" from-port="1" to-layer="23" to-port="6" />
895
+ <edge from-layer="22" from-port="2" to-layer="23" to-port="7" />
896
+ <edge from-layer="22" from-port="3" to-layer="23" to-port="8" />
897
+ <edge from-layer="23" from-port="9" to-layer="24" to-port="0" />
898
+ <edge from-layer="23" from-port="10" to-layer="24" to-port="1" />
899
+ <edge from-layer="23" from-port="11" to-layer="24" to-port="2" />
900
+ <edge from-layer="23" from-port="12" to-layer="24" to-port="3" />
901
+ <edge from-layer="23" from-port="13" to-layer="24" to-port="4" />
902
+ <edge from-layer="24" from-port="6" to-layer="45" to-port="1" />
903
+ <edge from-layer="24" from-port="7" to-layer="45" to-port="2" />
904
+ <edge from-layer="24" from-port="8" to-layer="45" to-port="3" />
905
+ <edge from-layer="24" from-port="9" to-layer="45" to-port="4" />
906
+ <edge from-layer="24" from-port="5" to-layer="45" to-port="0" />
907
+ <edge from-layer="25" from-port="0" to-layer="26" to-port="0" />
908
+ <edge from-layer="26" from-port="3" to-layer="45" to-port="7" />
909
+ <edge from-layer="26" from-port="2" to-layer="45" to-port="6" />
910
+ <edge from-layer="26" from-port="1" to-layer="45" to-port="5" />
911
+ <edge from-layer="27" from-port="0" to-layer="28" to-port="0" />
912
+ <edge from-layer="28" from-port="3" to-layer="45" to-port="10" />
913
+ <edge from-layer="28" from-port="2" to-layer="45" to-port="9" />
914
+ <edge from-layer="28" from-port="1" to-layer="45" to-port="8" />
915
+ <edge from-layer="29" from-port="0" to-layer="37" to-port="0" />
916
+ <edge from-layer="30" from-port="0" to-layer="31" to-port="0" />
917
+ <edge from-layer="31" from-port="1" to-layer="32" to-port="0" />
918
+ <edge from-layer="31" from-port="1" to-layer="43" to-port="2" />
919
+ <edge from-layer="31" from-port="2" to-layer="43" to-port="3" />
920
+ <edge from-layer="31" from-port="3" to-layer="43" to-port="4" />
921
+ <edge from-layer="32" from-port="1" to-layer="35" to-port="0" />
922
+ <edge from-layer="33" from-port="0" to-layer="35" to-port="1" />
923
+ <edge from-layer="34" from-port="0" to-layer="35" to-port="2" />
924
+ <edge from-layer="35" from-port="3" to-layer="40" to-port="0" />
925
+ <edge from-layer="35" from-port="3" to-layer="37" to-port="1" />
926
+ <edge from-layer="36" from-port="0" to-layer="37" to-port="2" />
927
+ <edge from-layer="37" from-port="3" to-layer="43" to-port="0" />
928
+ <edge from-layer="38" from-port="0" to-layer="42" to-port="0" />
929
+ <edge from-layer="39" from-port="0" to-layer="40" to-port="1" />
930
+ <edge from-layer="40" from-port="2" to-layer="42" to-port="1" />
931
+ <edge from-layer="41" from-port="0" to-layer="42" to-port="2" />
932
+ <edge from-layer="42" from-port="3" to-layer="43" to-port="1" />
933
+ <edge from-layer="43" from-port="8" to-layer="45" to-port="12" />
934
+ <edge from-layer="43" from-port="9" to-layer="45" to-port="13" />
935
+ <edge from-layer="43" from-port="7" to-layer="45" to-port="11" />
936
+ <edge from-layer="44" from-port="0" to-layer="45" to-port="14" />
937
+ <edge from-layer="45" from-port="16" to-layer="46" to-port="0" />
938
+ <edge from-layer="45" from-port="15" to-layer="46" to-port="1" />
939
+ <edge from-layer="45" from-port="16" to-layer="49" to-port="0" />
940
+ <edge from-layer="45" from-port="17" to-layer="54" to-port="2" />
941
+ <edge from-layer="45" from-port="16" to-layer="50" to-port="0" />
942
+ <edge from-layer="45" from-port="16" to-layer="54" to-port="1" />
943
+ <edge from-layer="46" from-port="2" to-layer="48" to-port="0" />
944
+ <edge from-layer="47" from-port="0" to-layer="48" to-port="1" />
945
+ <edge from-layer="48" from-port="2" to-layer="49" to-port="1" />
946
+ <edge from-layer="49" from-port="2" to-layer="54" to-port="0" />
947
+ <edge from-layer="49" from-port="2" to-layer="50" to-port="1" />
948
+ <edge from-layer="50" from-port="2" to-layer="52" to-port="0" />
949
+ <edge from-layer="51" from-port="0" to-layer="52" to-port="1" />
950
+ <edge from-layer="52" from-port="2" to-layer="54" to-port="3" />
951
+ <edge from-layer="53" from-port="0" to-layer="54" to-port="4" />
952
+ <edge from-layer="54" from-port="6" to-layer="55" to-port="0" />
953
+ <edge from-layer="54" from-port="5" to-layer="58" to-port="0" />
954
+ <edge from-layer="55" from-port="1" to-layer="56" to-port="0" />
955
+ <edge from-layer="56" from-port="1" to-layer="57" to-port="0" />
956
+ <edge from-layer="58" from-port="1" to-layer="59" to-port="0" />
957
+ </edges>
958
+ <rt_info>
959
+ <bos_token_id value="0" />
960
+ <chat_template value="{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'&lt;|im_start|>system&#10;' + system_message + '&lt;|im_end|>&#10;'}}{% endif %}{{'&lt;|im_start|>' + message['role'] + '&#10;' + message['content'] + '&lt;|im_end|>' + '&#10;'}}{% endfor %}{% if add_generation_prompt %}{{ '&lt;|im_start|>assistant&#10;' }}{% endif %}" />
961
+ <eos_token_id value="49152" />
962
+ <original_tokenizer_class value="&lt;class 'transformers.models.gpt2.tokenization_gpt2_fast.GPT2TokenizerFast'>" />
963
+ <pad_token_id value="0" />
964
+ </rt_info>
965
+ </net>
special_tokens_map.json ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|endoftext|>",
4
+ "<fim_prefix>",
5
+ "<fim_middle>",
6
+ "<fim_suffix>",
7
+ "<fim_pad>",
8
+ "<repo_name>",
9
+ "<file_sep>",
10
+ "<issue_start>",
11
+ "<issue_comment>",
12
+ "<issue_closed>",
13
+ "<jupyter_start>",
14
+ "<jupyter_text>",
15
+ "<jupyter_code>",
16
+ "<jupyter_output>",
17
+ "<jupyter_script>",
18
+ "<empty_output>",
19
+ "<code_to_intermediate>",
20
+ "<intermediate_to_code>",
21
+ "<pr>",
22
+ "<pr_status>",
23
+ "<pr_is_merged>",
24
+ "<pr_base>",
25
+ "<pr_file>",
26
+ "<pr_base_code>",
27
+ "<pr_diff>",
28
+ "<pr_diff_hunk>",
29
+ "<pr_comment>",
30
+ "<pr_event_id>",
31
+ "<pr_review>",
32
+ "<pr_review_state>",
33
+ "<pr_review_comment>",
34
+ "<pr_in_reply_to_review_id>",
35
+ "<pr_in_reply_to_comment_id>",
36
+ "<pr_diff_hunk_comment_line>",
37
+ "<NAME>",
38
+ "<EMAIL>",
39
+ "<KEY>",
40
+ "<PASSWORD>"
41
+ ],
42
+ "bos_token": {
43
+ "content": "<|endoftext|>",
44
+ "lstrip": false,
45
+ "normalized": false,
46
+ "rstrip": false,
47
+ "single_word": false
48
+ },
49
+ "eos_token": {
50
+ "content": "<|im_end|>",
51
+ "lstrip": false,
52
+ "normalized": false,
53
+ "rstrip": false,
54
+ "single_word": false
55
+ },
56
+ "pad_token": {
57
+ "content": "<|endoftext|>",
58
+ "lstrip": false,
59
+ "normalized": false,
60
+ "rstrip": false,
61
+ "single_word": false
62
+ },
63
+ "unk_token": {
64
+ "content": "<|endoftext|>",
65
+ "lstrip": false,
66
+ "normalized": false,
67
+ "rstrip": false,
68
+ "single_word": false
69
+ }
70
+ }
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,374 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "0": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "1": {
13
+ "content": "<fim_prefix>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "2": {
21
+ "content": "<fim_middle>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ },
28
+ "3": {
29
+ "content": "<fim_suffix>",
30
+ "lstrip": false,
31
+ "normalized": false,
32
+ "rstrip": false,
33
+ "single_word": false,
34
+ "special": true
35
+ },
36
+ "4": {
37
+ "content": "<fim_pad>",
38
+ "lstrip": false,
39
+ "normalized": false,
40
+ "rstrip": false,
41
+ "single_word": false,
42
+ "special": true
43
+ },
44
+ "5": {
45
+ "content": "<repo_name>",
46
+ "lstrip": false,
47
+ "normalized": false,
48
+ "rstrip": false,
49
+ "single_word": false,
50
+ "special": true
51
+ },
52
+ "6": {
53
+ "content": "<file_sep>",
54
+ "lstrip": false,
55
+ "normalized": false,
56
+ "rstrip": false,
57
+ "single_word": false,
58
+ "special": true
59
+ },
60
+ "7": {
61
+ "content": "<issue_start>",
62
+ "lstrip": false,
63
+ "normalized": false,
64
+ "rstrip": false,
65
+ "single_word": false,
66
+ "special": true
67
+ },
68
+ "8": {
69
+ "content": "<issue_comment>",
70
+ "lstrip": false,
71
+ "normalized": false,
72
+ "rstrip": false,
73
+ "single_word": false,
74
+ "special": true
75
+ },
76
+ "9": {
77
+ "content": "<issue_closed>",
78
+ "lstrip": false,
79
+ "normalized": false,
80
+ "rstrip": false,
81
+ "single_word": false,
82
+ "special": true
83
+ },
84
+ "10": {
85
+ "content": "<jupyter_start>",
86
+ "lstrip": false,
87
+ "normalized": false,
88
+ "rstrip": false,
89
+ "single_word": false,
90
+ "special": true
91
+ },
92
+ "11": {
93
+ "content": "<jupyter_text>",
94
+ "lstrip": false,
95
+ "normalized": false,
96
+ "rstrip": false,
97
+ "single_word": false,
98
+ "special": true
99
+ },
100
+ "12": {
101
+ "content": "<jupyter_code>",
102
+ "lstrip": false,
103
+ "normalized": false,
104
+ "rstrip": false,
105
+ "single_word": false,
106
+ "special": true
107
+ },
108
+ "13": {
109
+ "content": "<jupyter_output>",
110
+ "lstrip": false,
111
+ "normalized": false,
112
+ "rstrip": false,
113
+ "single_word": false,
114
+ "special": true
115
+ },
116
+ "14": {
117
+ "content": "<jupyter_script>",
118
+ "lstrip": false,
119
+ "normalized": false,
120
+ "rstrip": false,
121
+ "single_word": false,
122
+ "special": true
123
+ },
124
+ "15": {
125
+ "content": "<empty_output>",
126
+ "lstrip": false,
127
+ "normalized": false,
128
+ "rstrip": false,
129
+ "single_word": false,
130
+ "special": true
131
+ },
132
+ "16": {
133
+ "content": "<code_to_intermediate>",
134
+ "lstrip": false,
135
+ "normalized": false,
136
+ "rstrip": false,
137
+ "single_word": false,
138
+ "special": true
139
+ },
140
+ "17": {
141
+ "content": "<intermediate_to_code>",
142
+ "lstrip": false,
143
+ "normalized": false,
144
+ "rstrip": false,
145
+ "single_word": false,
146
+ "special": true
147
+ },
148
+ "18": {
149
+ "content": "<pr>",
150
+ "lstrip": false,
151
+ "normalized": false,
152
+ "rstrip": false,
153
+ "single_word": false,
154
+ "special": true
155
+ },
156
+ "19": {
157
+ "content": "<pr_status>",
158
+ "lstrip": false,
159
+ "normalized": false,
160
+ "rstrip": false,
161
+ "single_word": false,
162
+ "special": true
163
+ },
164
+ "20": {
165
+ "content": "<pr_is_merged>",
166
+ "lstrip": false,
167
+ "normalized": false,
168
+ "rstrip": false,
169
+ "single_word": false,
170
+ "special": true
171
+ },
172
+ "21": {
173
+ "content": "<pr_base>",
174
+ "lstrip": false,
175
+ "normalized": false,
176
+ "rstrip": false,
177
+ "single_word": false,
178
+ "special": true
179
+ },
180
+ "22": {
181
+ "content": "<pr_file>",
182
+ "lstrip": false,
183
+ "normalized": false,
184
+ "rstrip": false,
185
+ "single_word": false,
186
+ "special": true
187
+ },
188
+ "23": {
189
+ "content": "<pr_base_code>",
190
+ "lstrip": false,
191
+ "normalized": false,
192
+ "rstrip": false,
193
+ "single_word": false,
194
+ "special": true
195
+ },
196
+ "24": {
197
+ "content": "<pr_diff>",
198
+ "lstrip": false,
199
+ "normalized": false,
200
+ "rstrip": false,
201
+ "single_word": false,
202
+ "special": true
203
+ },
204
+ "25": {
205
+ "content": "<pr_diff_hunk>",
206
+ "lstrip": false,
207
+ "normalized": false,
208
+ "rstrip": false,
209
+ "single_word": false,
210
+ "special": true
211
+ },
212
+ "26": {
213
+ "content": "<pr_comment>",
214
+ "lstrip": false,
215
+ "normalized": false,
216
+ "rstrip": false,
217
+ "single_word": false,
218
+ "special": true
219
+ },
220
+ "27": {
221
+ "content": "<pr_event_id>",
222
+ "lstrip": false,
223
+ "normalized": false,
224
+ "rstrip": false,
225
+ "single_word": false,
226
+ "special": true
227
+ },
228
+ "28": {
229
+ "content": "<pr_review>",
230
+ "lstrip": false,
231
+ "normalized": false,
232
+ "rstrip": false,
233
+ "single_word": false,
234
+ "special": true
235
+ },
236
+ "29": {
237
+ "content": "<pr_review_state>",
238
+ "lstrip": false,
239
+ "normalized": false,
240
+ "rstrip": false,
241
+ "single_word": false,
242
+ "special": true
243
+ },
244
+ "30": {
245
+ "content": "<pr_review_comment>",
246
+ "lstrip": false,
247
+ "normalized": false,
248
+ "rstrip": false,
249
+ "single_word": false,
250
+ "special": true
251
+ },
252
+ "31": {
253
+ "content": "<pr_in_reply_to_review_id>",
254
+ "lstrip": false,
255
+ "normalized": false,
256
+ "rstrip": false,
257
+ "single_word": false,
258
+ "special": true
259
+ },
260
+ "32": {
261
+ "content": "<pr_in_reply_to_comment_id>",
262
+ "lstrip": false,
263
+ "normalized": false,
264
+ "rstrip": false,
265
+ "single_word": false,
266
+ "special": true
267
+ },
268
+ "33": {
269
+ "content": "<pr_diff_hunk_comment_line>",
270
+ "lstrip": false,
271
+ "normalized": false,
272
+ "rstrip": false,
273
+ "single_word": false,
274
+ "special": true
275
+ },
276
+ "34": {
277
+ "content": "<NAME>",
278
+ "lstrip": false,
279
+ "normalized": false,
280
+ "rstrip": false,
281
+ "single_word": false,
282
+ "special": true
283
+ },
284
+ "35": {
285
+ "content": "<EMAIL>",
286
+ "lstrip": false,
287
+ "normalized": false,
288
+ "rstrip": false,
289
+ "single_word": false,
290
+ "special": true
291
+ },
292
+ "36": {
293
+ "content": "<KEY>",
294
+ "lstrip": false,
295
+ "normalized": false,
296
+ "rstrip": false,
297
+ "single_word": false,
298
+ "special": true
299
+ },
300
+ "37": {
301
+ "content": "<PASSWORD>",
302
+ "lstrip": false,
303
+ "normalized": false,
304
+ "rstrip": false,
305
+ "single_word": false,
306
+ "special": true
307
+ },
308
+ "49152": {
309
+ "content": "<|im_end|>",
310
+ "lstrip": false,
311
+ "normalized": false,
312
+ "rstrip": false,
313
+ "single_word": false,
314
+ "special": true
315
+ },
316
+ "49153": {
317
+ "content": "<|im_start|>",
318
+ "lstrip": false,
319
+ "normalized": false,
320
+ "rstrip": false,
321
+ "single_word": false,
322
+ "special": false
323
+ }
324
+ },
325
+ "additional_special_tokens": [
326
+ "<|endoftext|>",
327
+ "<fim_prefix>",
328
+ "<fim_middle>",
329
+ "<fim_suffix>",
330
+ "<fim_pad>",
331
+ "<repo_name>",
332
+ "<file_sep>",
333
+ "<issue_start>",
334
+ "<issue_comment>",
335
+ "<issue_closed>",
336
+ "<jupyter_start>",
337
+ "<jupyter_text>",
338
+ "<jupyter_code>",
339
+ "<jupyter_output>",
340
+ "<jupyter_script>",
341
+ "<empty_output>",
342
+ "<code_to_intermediate>",
343
+ "<intermediate_to_code>",
344
+ "<pr>",
345
+ "<pr_status>",
346
+ "<pr_is_merged>",
347
+ "<pr_base>",
348
+ "<pr_file>",
349
+ "<pr_base_code>",
350
+ "<pr_diff>",
351
+ "<pr_diff_hunk>",
352
+ "<pr_comment>",
353
+ "<pr_event_id>",
354
+ "<pr_review>",
355
+ "<pr_review_state>",
356
+ "<pr_review_comment>",
357
+ "<pr_in_reply_to_review_id>",
358
+ "<pr_in_reply_to_comment_id>",
359
+ "<pr_diff_hunk_comment_line>",
360
+ "<NAME>",
361
+ "<EMAIL>",
362
+ "<KEY>",
363
+ "<PASSWORD>"
364
+ ],
365
+ "bos_token": "<|endoftext|>",
366
+ "chat_template": "{% if messages[0]['role'] == 'system' %}{% set loop_messages = messages[1:] %}{% set system_message = messages[0]['content'] %}{% else %}{% set loop_messages = messages %}{% set system_message = 'You are a helpful assistant.' %}{% endif %}{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in loop_messages %}{% if loop.index0 == 0 %}{{'<|im_start|>system\n' + system_message + '<|im_end|>\n'}}{% endif %}{{'<|im_start|>' + message['role'] + '\n' + message['content'] + '<|im_end|>' + '\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\n' }}{% endif %}",
367
+ "clean_up_tokenization_spaces": true,
368
+ "eos_token": "<|im_end|>",
369
+ "model_max_length": 1000000000000000019884624838656,
370
+ "pad_token": "<|endoftext|>",
371
+ "tokenizer_class": "GPT2Tokenizer",
372
+ "unk_token": "<|endoftext|>",
373
+ "vocab_size": 49152
374
+ }
vocab.json ADDED
The diff for this file is too large to render. See raw diff