Hugging Face's logo

KoichiYasuoka
/
Xunzi-Qwen2-7B-upos

Token Classification
Transformers
PyTorch
Literary Chinese
qwen2
classical chinese
literary chinese
ancient chinese
pos
dependency-parsing
text-generation-inference
Inference Endpoints
Model card Files Community
KoichiYasuoka commited on
Commit
6d64613
•
1 Parent(s): 8c04955

initial release

Browse files
Files changed (17) hide show
  1. README.md +31 -0
  2. config.json +374 -0
  3. maker.sh +105 -0
  4. mergeout.sh +3 -0
  5. merges.txt +0 -0
  6. oldmerges.txt +0 -0
  7. pytorch_model-00001-of-00006.bin +3 -0
  8. pytorch_model-00002-of-00006.bin +3 -0
  9. pytorch_model-00003-of-00006.bin +3 -0
  10. pytorch_model-00004-of-00006.bin +3 -0
  11. pytorch_model-00005-of-00006.bin +3 -0
  12. pytorch_model-00006-of-00006.bin +3 -0
  13. pytorch_model.bin.index.json +347 -0
  14. special_tokens_map.json +20 -0
  15. tokenizer_config.json +44 -0
  16. upos.py +41 -0
  17. vocab.json +0 -0
README.md ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ language:
3
+ - "lzh"
4
+ tags:
5
+ - "classical chinese"
6
+ - "literary chinese"
7
+ - "ancient chinese"
8
+ - "token-classification"
9
+ - "pos"
10
+ - "dependency-parsing"
11
+ datasets:
12
+ - "universal_dependencies"
13
+ license: "apache-2.0"
14
+ pipeline_tag: "token-classification"
15
+ widget:
16
+ - text: "子曰學而時習之不亦説乎有朋自遠方來不亦樂乎人不知而不慍不亦君子乎"
17
+ ---
18
+
19
+ # Xunzi-Qwen2-7B-upos
20
+
21
+ ## Model Description
22
+
23
+ This is a Qwen2 model pre-trained on Classical Chinese texts for POS-tagging, derived from [Xunzi-Qwen2-7B](https://www.modelscope.cn/models/Xunzillm4cc/Xunzi-Qwen2-7B). Every word is tagged by [UPOS](https://universaldependencies.org/u/pos/) (Universal Part-Of-Speech) and [FEATS](https://universaldependencies.org/u/feat/).
24
+
25
+ ## How to Use
26
+
27
+ ```py
28
+ from transformers import pipeline
29
+ nlp=pipeline("upos","KoichiYasuoka/Xunzi-Qwen2-7B-upos",trust_remote_code=True,aggregation_strategy="simple")
30
+ print(nlp("不入虎穴不得虎子"))
31
+ ```
config.json ADDED
@@ -0,0 +1,374 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "Qwen2ForTokenClassification"
4
+ ],
5
+ "attention_dropout": 0.0,
6
+ "bos_token_id": 151643,
7
+ "custom_pipelines": {
8
+ "upos": {
9
+ "impl": "upos.BellmanFordTokenClassificationPipeline",
10
+ "pt": "AutoModelForTokenClassification"
11
+ }
12
+ },
13
+ "eos_token_id": 151643,
14
+ "hidden_act": "silu",
15
+ "hidden_size": 3584,
16
+ "id2label": {
17
+ "0": "ADP",
18
+ "1": "ADP|Degree=Equ",
19
+ "2": "ADV",
20
+ "3": "ADV|AdvType=Cau",
21
+ "4": "ADV|AdvType=Deg|Degree=Cmp",
22
+ "5": "ADV|AdvType=Deg|Degree=Pos",
23
+ "6": "ADV|AdvType=Deg|Degree=Sup",
24
+ "7": "ADV|AdvType=Tim",
25
+ "8": "ADV|AdvType=Tim|Aspect=Perf",
26
+ "9": "ADV|AdvType=Tim|Tense=Fut",
27
+ "10": "ADV|AdvType=Tim|Tense=Past",
28
+ "11": "ADV|AdvType=Tim|Tense=Pres",
29
+ "12": "ADV|Degree=Equ|VerbForm=Conv",
30
+ "13": "ADV|Degree=Pos|VerbForm=Conv",
31
+ "14": "ADV|Polarity=Neg",
32
+ "15": "ADV|Polarity=Neg|VerbForm=Conv",
33
+ "16": "ADV|VerbForm=Conv",
34
+ "17": "AUX|Mood=Des",
35
+ "18": "AUX|Mood=Nec",
36
+ "19": "AUX|Mood=Pot",
37
+ "20": "AUX|VerbType=Cop",
38
+ "21": "AUX|Voice=Pass",
39
+ "22": "B-ADP",
40
+ "23": "B-ADP|Degree=Equ",
41
+ "24": "B-ADV",
42
+ "25": "B-ADV|AdvType=Cau",
43
+ "26": "B-ADV|AdvType=Deg|Degree=Cmp",
44
+ "27": "B-ADV|AdvType=Deg|Degree=Pos",
45
+ "28": "B-ADV|AdvType=Deg|Degree=Sup",
46
+ "29": "B-ADV|AdvType=Tim",
47
+ "30": "B-ADV|AdvType=Tim|Aspect=Perf",
48
+ "31": "B-ADV|AdvType=Tim|Tense=Fut",
49
+ "32": "B-ADV|AdvType=Tim|Tense=Past",
50
+ "33": "B-ADV|AdvType=Tim|Tense=Pres",
51
+ "34": "B-ADV|Degree=Equ|VerbForm=Conv",
52
+ "35": "B-ADV|Degree=Pos|VerbForm=Conv",
53
+ "36": "B-ADV|Polarity=Neg",
54
+ "37": "B-ADV|Polarity=Neg|VerbForm=Conv",
55
+ "38": "B-ADV|VerbForm=Conv",
56
+ "39": "B-AUX|Mood=Des",
57
+ "40": "B-AUX|Mood=Nec",
58
+ "41": "B-AUX|Mood=Pot",
59
+ "42": "B-AUX|VerbType=Cop",
60
+ "43": "B-AUX|Voice=Pass",
61
+ "44": "B-CCONJ",
62
+ "45": "B-INTJ",
63
+ "46": "B-NOUN",
64
+ "47": "B-NOUN|Case=Loc",
65
+ "48": "B-NOUN|Case=Tem",
66
+ "49": "B-NOUN|Degree=Pos",
67
+ "50": "B-NOUN|NounType=Clf",
68
+ "51": "B-NUM",
69
+ "52": "B-NUM|NumType=Ord",
70
+ "53": "B-PART",
71
+ "54": "B-PRON|Person=1|PronType=Prs",
72
+ "55": "B-PRON|Person=2|PronType=Prs",
73
+ "56": "B-PRON|Person=3|PronType=Prs",
74
+ "57": "B-PRON|PronType=Dem",
75
+ "58": "B-PRON|PronType=Int",
76
+ "59": "B-PRON|PronType=Prs",
77
+ "60": "B-PRON|PronType=Prs|Reflex=Yes",
78
+ "61": "B-PROPN",
79
+ "62": "B-PROPN|Case=Loc|NameType=Geo",
80
+ "63": "B-PROPN|Case=Loc|NameType=Nat",
81
+ "64": "B-PROPN|NameType=Giv",
82
+ "65": "B-PROPN|NameType=Prs",
83
+ "66": "B-PROPN|NameType=Sur",
84
+ "67": "B-PUNCT",
85
+ "68": "B-SCONJ",
86
+ "69": "B-SYM",
87
+ "70": "B-VERB",
88
+ "71": "B-VERB|Degree=Equ",
89
+ "72": "B-VERB|Degree=Equ|VerbForm=Part",
90
+ "73": "B-VERB|Degree=Pos",
91
+ "74": "B-VERB|Degree=Pos|VerbForm=Part",
92
+ "75": "B-VERB|Polarity=Neg",
93
+ "76": "B-VERB|Polarity=Neg|VerbForm=Part",
94
+ "77": "B-VERB|VerbForm=Part",
95
+ "78": "CCONJ",
96
+ "79": "I-ADP",
97
+ "80": "I-ADP|Degree=Equ",
98
+ "81": "I-ADV",
99
+ "82": "I-ADV|AdvType=Cau",
100
+ "83": "I-ADV|AdvType=Deg|Degree=Cmp",
101
+ "84": "I-ADV|AdvType=Deg|Degree=Pos",
102
+ "85": "I-ADV|AdvType=Deg|Degree=Sup",
103
+ "86": "I-ADV|AdvType=Tim",
104
+ "87": "I-ADV|AdvType=Tim|Aspect=Perf",
105
+ "88": "I-ADV|AdvType=Tim|Tense=Fut",
106
+ "89": "I-ADV|AdvType=Tim|Tense=Past",
107
+ "90": "I-ADV|AdvType=Tim|Tense=Pres",
108
+ "91": "I-ADV|Degree=Equ|VerbForm=Conv",
109
+ "92": "I-ADV|Degree=Pos|VerbForm=Conv",
110
+ "93": "I-ADV|Polarity=Neg",
111
+ "94": "I-ADV|Polarity=Neg|VerbForm=Conv",
112
+ "95": "I-ADV|VerbForm=Conv",
113
+ "96": "I-AUX|Mood=Des",
114
+ "97": "I-AUX|Mood=Nec",
115
+ "98": "I-AUX|Mood=Pot",
116
+ "99": "I-AUX|VerbType=Cop",
117
+ "100": "I-AUX|Voice=Pass",
118
+ "101": "I-CCONJ",
119
+ "102": "I-INTJ",
120
+ "103": "I-NOUN",
121
+ "104": "I-NOUN|Case=Loc",
122
+ "105": "I-NOUN|Case=Tem",
123
+ "106": "I-NOUN|Degree=Pos",
124
+ "107": "I-NOUN|NounType=Clf",
125
+ "108": "I-NUM",
126
+ "109": "I-NUM|NumType=Ord",
127
+ "110": "I-PART",
128
+ "111": "I-PRON|Person=1|PronType=Prs",
129
+ "112": "I-PRON|Person=2|PronType=Prs",
130
+ "113": "I-PRON|Person=3|PronType=Prs",
131
+ "114": "I-PRON|PronType=Dem",
132
+ "115": "I-PRON|PronType=Int",
133
+ "116": "I-PRON|PronType=Prs",
134
+ "117": "I-PRON|PronType=Prs|Reflex=Yes",
135
+ "118": "I-PROPN",
136
+ "119": "I-PROPN|Case=Loc|NameType=Geo",
137
+ "120": "I-PROPN|Case=Loc|NameType=Nat",
138
+ "121": "I-PROPN|NameType=Giv",
139
+ "122": "I-PROPN|NameType=Prs",
140
+ "123": "I-PROPN|NameType=Sur",
141
+ "124": "I-PUNCT",
142
+ "125": "I-SCONJ",
143
+ "126": "I-SYM",
144
+ "127": "I-VERB",
145
+ "128": "I-VERB|Degree=Equ",
146
+ "129": "I-VERB|Degree=Equ|VerbForm=Part",
147
+ "130": "I-VERB|Degree=Pos",
148
+ "131": "I-VERB|Degree=Pos|VerbForm=Part",
149
+ "132": "I-VERB|Polarity=Neg",
150
+ "133": "I-VERB|Polarity=Neg|VerbForm=Part",
151
+ "134": "I-VERB|VerbForm=Part",
152
+ "135": "INTJ",
153
+ "136": "NOUN",
154
+ "137": "NOUN|Case=Loc",
155
+ "138": "NOUN|Case=Tem",
156
+ "139": "NOUN|Degree=Pos",
157
+ "140": "NOUN|NounType=Clf",
158
+ "141": "NUM",
159
+ "142": "NUM|NumType=Ord",
160
+ "143": "PART",
161
+ "144": "PRON|Person=1|PronType=Prs",
162
+ "145": "PRON|Person=2|PronType=Prs",
163
+ "146": "PRON|Person=3|PronType=Prs",
164
+ "147": "PRON|PronType=Dem",
165
+ "148": "PRON|PronType=Int",
166
+ "149": "PRON|PronType=Prs",
167
+ "150": "PRON|PronType=Prs|Reflex=Yes",
168
+ "151": "PROPN",
169
+ "152": "PROPN|Case=Loc|NameType=Geo",
170
+ "153": "PROPN|Case=Loc|NameType=Nat",
171
+ "154": "PROPN|NameType=Giv",
172
+ "155": "PROPN|NameType=Prs",
173
+ "156": "PROPN|NameType=Sur",
174
+ "157": "PUNCT",
175
+ "158": "SCONJ",
176
+ "159": "SYM",
177
+ "160": "VERB",
178
+ "161": "VERB|Degree=Equ",
179
+ "162": "VERB|Degree=Equ|VerbForm=Part",
180
+ "163": "VERB|Degree=Pos",
181
+ "164": "VERB|Degree=Pos|VerbForm=Part",
182
+ "165": "VERB|Polarity=Neg",
183
+ "166": "VERB|Polarity=Neg|VerbForm=Part",
184
+ "167": "VERB|VerbForm=Part"
185
+ },
186
+ "initializer_range": 0.02,
187
+ "intermediate_size": 18944,
188
+ "label2id": {
189
+ "ADP": 0,
190
+ "ADP|Degree=Equ": 1,
191
+ "ADV": 2,
192
+ "ADV|AdvType=Cau": 3,
193
+ "ADV|AdvType=Deg|Degree=Cmp": 4,
194
+ "ADV|AdvType=Deg|Degree=Pos": 5,
195
+ "ADV|AdvType=Deg|Degree=Sup": 6,
196
+ "ADV|AdvType=Tim": 7,
197
+ "ADV|AdvType=Tim|Aspect=Perf": 8,
198
+ "ADV|AdvType=Tim|Tense=Fut": 9,
199
+ "ADV|AdvType=Tim|Tense=Past": 10,
200
+ "ADV|AdvType=Tim|Tense=Pres": 11,
201
+ "ADV|Degree=Equ|VerbForm=Conv": 12,
202
+ "ADV|Degree=Pos|VerbForm=Conv": 13,
203
+ "ADV|Polarity=Neg": 14,
204
+ "ADV|Polarity=Neg|VerbForm=Conv": 15,
205
+ "ADV|VerbForm=Conv": 16,
206
+ "AUX|Mood=Des": 17,
207
+ "AUX|Mood=Nec": 18,
208
+ "AUX|Mood=Pot": 19,
209
+ "AUX|VerbType=Cop": 20,
210
+ "AUX|Voice=Pass": 21,
211
+ "B-ADP": 22,
212
+ "B-ADP|Degree=Equ": 23,
213
+ "B-ADV": 24,
214
+ "B-ADV|AdvType=Cau": 25,
215
+ "B-ADV|AdvType=Deg|Degree=Cmp": 26,
216
+ "B-ADV|AdvType=Deg|Degree=Pos": 27,
217
+ "B-ADV|AdvType=Deg|Degree=Sup": 28,
218
+ "B-ADV|AdvType=Tim": 29,
219
+ "B-ADV|AdvType=Tim|Aspect=Perf": 30,
220
+ "B-ADV|AdvType=Tim|Tense=Fut": 31,
221
+ "B-ADV|AdvType=Tim|Tense=Past": 32,
222
+ "B-ADV|AdvType=Tim|Tense=Pres": 33,
223
+ "B-ADV|Degree=Equ|VerbForm=Conv": 34,
224
+ "B-ADV|Degree=Pos|VerbForm=Conv": 35,
225
+ "B-ADV|Polarity=Neg": 36,
226
+ "B-ADV|Polarity=Neg|VerbForm=Conv": 37,
227
+ "B-ADV|VerbForm=Conv": 38,
228
+ "B-AUX|Mood=Des": 39,
229
+ "B-AUX|Mood=Nec": 40,
230
+ "B-AUX|Mood=Pot": 41,
231
+ "B-AUX|VerbType=Cop": 42,
232
+ "B-AUX|Voice=Pass": 43,
233
+ "B-CCONJ": 44,
234
+ "B-INTJ": 45,
235
+ "B-NOUN": 46,
236
+ "B-NOUN|Case=Loc": 47,
237
+ "B-NOUN|Case=Tem": 48,
238
+ "B-NOUN|Degree=Pos": 49,
239
+ "B-NOUN|NounType=Clf": 50,
240
+ "B-NUM": 51,
241
+ "B-NUM|NumType=Ord": 52,
242
+ "B-PART": 53,
243
+ "B-PRON|Person=1|PronType=Prs": 54,
244
+ "B-PRON|Person=2|PronType=Prs": 55,
245
+ "B-PRON|Person=3|PronType=Prs": 56,
246
+ "B-PRON|PronType=Dem": 57,
247
+ "B-PRON|PronType=Int": 58,
248
+ "B-PRON|PronType=Prs": 59,
249
+ "B-PRON|PronType=Prs|Reflex=Yes": 60,
250
+ "B-PROPN": 61,
251
+ "B-PROPN|Case=Loc|NameType=Geo": 62,
252
+ "B-PROPN|Case=Loc|NameType=Nat": 63,
253
+ "B-PROPN|NameType=Giv": 64,
254
+ "B-PROPN|NameType=Prs": 65,
255
+ "B-PROPN|NameType=Sur": 66,
256
+ "B-PUNCT": 67,
257
+ "B-SCONJ": 68,
258
+ "B-SYM": 69,
259
+ "B-VERB": 70,
260
+ "B-VERB|Degree=Equ": 71,
261
+ "B-VERB|Degree=Equ|VerbForm=Part": 72,
262
+ "B-VERB|Degree=Pos": 73,
263
+ "B-VERB|Degree=Pos|VerbForm=Part": 74,
264
+ "B-VERB|Polarity=Neg": 75,
265
+ "B-VERB|Polarity=Neg|VerbForm=Part": 76,
266
+ "B-VERB|VerbForm=Part": 77,
267
+ "CCONJ": 78,
268
+ "I-ADP": 79,
269
+ "I-ADP|Degree=Equ": 80,
270
+ "I-ADV": 81,
271
+ "I-ADV|AdvType=Cau": 82,
272
+ "I-ADV|AdvType=Deg|Degree=Cmp": 83,
273
+ "I-ADV|AdvType=Deg|Degree=Pos": 84,
274
+ "I-ADV|AdvType=Deg|Degree=Sup": 85,
275
+ "I-ADV|AdvType=Tim": 86,
276
+ "I-ADV|AdvType=Tim|Aspect=Perf": 87,
277
+ "I-ADV|AdvType=Tim|Tense=Fut": 88,
278
+ "I-ADV|AdvType=Tim|Tense=Past": 89,
279
+ "I-ADV|AdvType=Tim|Tense=Pres": 90,
280
+ "I-ADV|Degree=Equ|VerbForm=Conv": 91,
281
+ "I-ADV|Degree=Pos|VerbForm=Conv": 92,
282
+ "I-ADV|Polarity=Neg": 93,
283
+ "I-ADV|Polarity=Neg|VerbForm=Conv": 94,
284
+ "I-ADV|VerbForm=Conv": 95,
285
+ "I-AUX|Mood=Des": 96,
286
+ "I-AUX|Mood=Nec": 97,
287
+ "I-AUX|Mood=Pot": 98,
288
+ "I-AUX|VerbType=Cop": 99,
289
+ "I-AUX|Voice=Pass": 100,
290
+ "I-CCONJ": 101,
291
+ "I-INTJ": 102,
292
+ "I-NOUN": 103,
293
+ "I-NOUN|Case=Loc": 104,
294
+ "I-NOUN|Case=Tem": 105,
295
+ "I-NOUN|Degree=Pos": 106,
296
+ "I-NOUN|NounType=Clf": 107,
297
+ "I-NUM": 108,
298
+ "I-NUM|NumType=Ord": 109,
299
+ "I-PART": 110,
300
+ "I-PRON|Person=1|PronType=Prs": 111,
301
+ "I-PRON|Person=2|PronType=Prs": 112,
302
+ "I-PRON|Person=3|PronType=Prs": 113,
303
+ "I-PRON|PronType=Dem": 114,
304
+ "I-PRON|PronType=Int": 115,
305
+ "I-PRON|PronType=Prs": 116,
306
+ "I-PRON|PronType=Prs|Reflex=Yes": 117,
307
+ "I-PROPN": 118,
308
+ "I-PROPN|Case=Loc|NameType=Geo": 119,
309
+ "I-PROPN|Case=Loc|NameType=Nat": 120,
310
+ "I-PROPN|NameType=Giv": 121,
311
+ "I-PROPN|NameType=Prs": 122,
312
+ "I-PROPN|NameType=Sur": 123,
313
+ "I-PUNCT": 124,
314
+ "I-SCONJ": 125,
315
+ "I-SYM": 126,
316
+ "I-VERB": 127,
317
+ "I-VERB|Degree=Equ": 128,
318
+ "I-VERB|Degree=Equ|VerbForm=Part": 129,
319
+ "I-VERB|Degree=Pos": 130,
320
+ "I-VERB|Degree=Pos|VerbForm=Part": 131,
321
+ "I-VERB|Polarity=Neg": 132,
322
+ "I-VERB|Polarity=Neg|VerbForm=Part": 133,
323
+ "I-VERB|VerbForm=Part": 134,
324
+ "INTJ": 135,
325
+ "NOUN": 136,
326
+ "NOUN|Case=Loc": 137,
327
+ "NOUN|Case=Tem": 138,
328
+ "NOUN|Degree=Pos": 139,
329
+ "NOUN|NounType=Clf": 140,
330
+ "NUM": 141,
331
+ "NUM|NumType=Ord": 142,
332
+ "PART": 143,
333
+ "PRON|Person=1|PronType=Prs": 144,
334
+ "PRON|Person=2|PronType=Prs": 145,
335
+ "PRON|Person=3|PronType=Prs": 146,
336
+ "PRON|PronType=Dem": 147,
337
+ "PRON|PronType=Int": 148,
338
+ "PRON|PronType=Prs": 149,
339
+ "PRON|PronType=Prs|Reflex=Yes": 150,
340
+ "PROPN": 151,
341
+ "PROPN|Case=Loc|NameType=Geo": 152,
342
+ "PROPN|Case=Loc|NameType=Nat": 153,
343
+ "PROPN|NameType=Giv": 154,
344
+ "PROPN|NameType=Prs": 155,
345
+ "PROPN|NameType=Sur": 156,
346
+ "PUNCT": 157,
347
+ "SCONJ": 158,
348
+ "SYM": 159,
349
+ "VERB": 160,
350
+ "VERB|Degree=Equ": 161,
351
+ "VERB|Degree=Equ|VerbForm=Part": 162,
352
+ "VERB|Degree=Pos": 163,
353
+ "VERB|Degree=Pos|VerbForm=Part": 164,
354
+ "VERB|Polarity=Neg": 165,
355
+ "VERB|Polarity=Neg|VerbForm=Part": 166,
356
+ "VERB|VerbForm=Part": 167
357
+ },
358
+ "max_position_embeddings": 131072,
359
+ "max_window_layers": 28,
360
+ "model_type": "qwen2",
361
+ "num_attention_heads": 28,
362
+ "num_hidden_layers": 28,
363
+ "num_key_value_heads": 4,
364
+ "rms_norm_eps": 1e-06,
365
+ "rope_theta": 1000000.0,
366
+ "sliding_window": 131072,
367
+ "tie_word_embeddings": false,
368
+ "tokenizer_class": "Qwen2Tokenizer",
369
+ "torch_dtype": "float32",
370
+ "transformers_version": "4.42.4",
371
+ "use_cache": false,
372
+ "use_sliding_window": false,
373
+ "vocab_size": 152064
374
+ }
maker.sh ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #! /bin/sh
2
+ S=Xunzi-Qwen2-7B
3
+ U=UD_Classical_Chinese-Kyoto
4
+ test -d $U || git clone --depth=1 https://github.com/UniversalDependencies/$U
5
+ for F in train dev test
6
+ do cp $U/*-$F.conllu $F.conllu
7
+ done
8
+ test -d $S || git clone --depth=1 https://www.modelscope.cn/Xunzillm4cc/$S.git
9
+
10
+ TMP=./maker$$.py
11
+ ( echo '#! /usr/bin/env deepspeed'
12
+ echo 'src="'$S'"'
13
+ echo 'tgt="KoichiYasuoka/'$S'-upos"'
14
+ ) > $TMP
15
+ cat << 'EOF' >> $TMP
16
+ from transformers import AutoTokenizer,Qwen2ForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
17
+
18
+ class UPOSFileDataset(object):
19
+ def __init__(self,conllu,tokenizer):
20
+ self.conllu=open(conllu,"r",encoding="utf-8")
21
+ self.tokenizer=tokenizer
22
+ self.seeks=[0]
23
+ self.multiword={}
24
+ label=set(["SYM"])
25
+ s=self.conllu.readline()
26
+ while s!="":
27
+ if s=="\n":
28
+ self.seeks.append(self.conllu.tell())
29
+ else:
30
+ w=s.split("\t")
31
+ if len(w)==10:
32
+ if w[0].isdecimal():
33
+ label.add(w[3] if w[5]=="_" else w[3]+"|"+w[5])
34
+ elif w[0].find("-")>0:
35
+ t=w[0].split("-")
36
+ f,j,k=w[1],[],[]
37
+ for i in range(int(t[0]),int(t[1])+1):
38
+ w=self.conllu.readline().split("\t")
39
+ j.append(w[3] if w[5]=="_" else w[3]+"|"+w[5])
40
+ k.append(w[1])
41
+ p="+".join(j)
42
+ label.add(p)
43
+ if p in self.multiword:
44
+ self.multiword[p][f]=list(k)
45
+ else:
46
+ self.multiword[p]={f:list(k)}
47
+ s=self.conllu.readline()
48
+ lid={}
49
+ for i,l in enumerate(sorted(label)):
50
+ lid[l],lid["B-"+l],lid["I-"+l]=i*3,i*3+1,i*3+2
51
+ self.label2id=lid
52
+ def __call__(*args):
53
+ lid={l:i for i,l in enumerate(sorted(set(sum([list(t.label2id) for t in args],[]))))}
54
+ for t in args:
55
+ t.label2id=lid
56
+ return lid
57
+ def __del__(self):
58
+ self.conllu.close()
59
+ __len__=lambda self:len(self.seeks)-1
60
+ def __getitem__(self,i):
61
+ self.conllu.seek(self.seeks[i])
62
+ form,upos=[],[]
63
+ while self.conllu.tell()<self.seeks[i+1]:
64
+ w=self.conllu.readline().split("\t")
65
+ if len(w)==10:
66
+ form.append(w[1])
67
+ if w[0].isdecimal():
68
+ upos.append(w[3] if w[5]=="_" else w[3]+"|"+w[5])
69
+ elif w[0].find("-")>0:
70
+ t=w[0].split("-")
71
+ u=[]
72
+ for j in range(int(t[0]),int(t[1])+1):
73
+ k=self.conllu.readline().split("\t")
74
+ u.append(k[3] if k[5]=="_" else k[3]+"|"+k[5])
75
+ upos.append("+".join(u))
76
+ v=self.tokenizer(form,add_special_tokens=False)
77
+ i,u=[],[]
78
+ for j,(x,y) in enumerate(zip(v["input_ids"],upos)):
79
+ if x!=[]:
80
+ i+=x
81
+ u+=[y] if len(x)==1 else ["B-"+y]+["I-"+y]*(len(x)-1)
82
+ if len(i)<self.tokenizer.model_max_length-3:
83
+ ids=i
84
+ upos=u
85
+ else:
86
+ ids=i[0:self.tokenizer.model_max_length-2]
87
+ upos=u[0:self.tokenizer.model_max_length-2]
88
+ return {"input_ids":ids,"labels":[self.label2id[t] for t in upos]}
89
+
90
+ tkz=AutoTokenizer.from_pretrained(src)
91
+ trainDS=UPOSFileDataset("train.conllu",tkz)
92
+ devDS=UPOSFileDataset("dev.conllu",tkz)
93
+ testDS=UPOSFileDataset("test.conllu",tkz)
94
+ lid=trainDS(devDS,testDS)
95
+ cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()},ignore_mismatched_sizes=True)
96
+ dsp={"fp16":{"enabled":"auto"},"optimizer":{"type":"AdamW"},"scheduler":{"type":"WarmupLR","params":{}},"train_batch_size":"auto","train_micro_batch_size_per_gpu":"auto","zero_optimization":{"stage":3,"offload_optimizer":{"device":"cpu","pin_memory":True},"offload_param":{"device":"cpu","pin_memory":True},"overlap_comm":True,"contiguous_gradients":True,"reduce_bucket_size":"auto","stage3_prefetch_bucket_size":"auto","stage3_param_persistence_threshold":"auto","stage3_gather_16bit_weights_on_model_save":True}}
97
+ arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=16,deepspeed=dsp,output_dir=tgt,overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1,save_safetensors=False)
98
+ trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=Qwen2ForTokenClassification.from_pretrained(src,config=cfg,ignore_mismatched_sizes=True),train_dataset=trainDS)
99
+ trn.train()
100
+ trn.save_model(tgt)
101
+ tkz.save_pretrained(tgt)
102
+ EOF
103
+ chmod 755 $TMP
104
+ $TMP
105
+ exit
mergeout.sh ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ #! /usr/bin/egrep -vf
2
+ ^ã[^ĢģĤĥĦħĨĩĪīĬĭĮįİı ][^ ]
3
+ ^[ãäåæçèé][^ ][^ ]
merges.txt ADDED
The diff for this file is too large to render. See raw diff
 
oldmerges.txt ADDED
The diff for this file is too large to render. See raw diff
 
pytorch_model-00001-of-00006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:1244798894da0c763c76601300132199d9efd4c03258f2fa84b8d5cb1d7bcb93
3
+ size 4976695876
pytorch_model-00002-of-00006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5ee0435a5b464f12b48d0fde09dc930a97beb337cb416969d944034b2926a475
3
+ size 4778637580
pytorch_model-00003-of-00006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6274f1d690bff436e926e46603b7ce4d81cd9de98d4be754533a0411ee06d054
3
+ size 4932758012
pytorch_model-00004-of-00006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:e4b5f23f228f5f49bbc7255da4fab124a5fd290891bf14d903bacdf07c1a2ee6
3
+ size 4932758052
pytorch_model-00005-of-00006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:39f3202969203fb4af3402becbdd6d25a1a3d366e04168ee813c631500a3bf7f
3
+ size 4998868108
pytorch_model-00006-of-00006.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:49ca160abd1cf1138a3ac53b65d80b198d080ed92ac51b1a4f67d0d6a3ba7934
3
+ size 3665284756
pytorch_model.bin.index.json ADDED
@@ -0,0 +1,347 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "metadata": {
3
+ "total_size": 28284885664
4
+ },
5
+ "weight_map": {
6
+ "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
7
+ "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
8
+ "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
9
+ "model.layers.0.mlp.gate_proj.weight": "pytorch_model-00001-of-00006.bin",
10
+ "model.layers.0.mlp.up_proj.weight": "pytorch_model-00001-of-00006.bin",
11
+ "model.layers.0.post_attention_layernorm.weight": "pytorch_model-00001-of-00006.bin",
12
+ "model.layers.0.self_attn.k_proj.bias": "pytorch_model-00001-of-00006.bin",
13
+ "model.layers.0.self_attn.k_proj.weight": "pytorch_model-00001-of-00006.bin",
14
+ "model.layers.0.self_attn.o_proj.weight": "pytorch_model-00001-of-00006.bin",
15
+ "model.layers.0.self_attn.q_proj.bias": "pytorch_model-00001-of-00006.bin",
16
+ "model.layers.0.self_attn.q_proj.weight": "pytorch_model-00001-of-00006.bin",
17
+ "model.layers.0.self_attn.v_proj.bias": "pytorch_model-00001-of-00006.bin",
18
+ "model.layers.0.self_attn.v_proj.weight": "pytorch_model-00001-of-00006.bin",
19
+ "model.layers.1.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
20
+ "model.layers.1.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
21
+ "model.layers.1.mlp.gate_proj.weight": "pytorch_model-00001-of-00006.bin",
22
+ "model.layers.1.mlp.up_proj.weight": "pytorch_model-00001-of-00006.bin",
23
+ "model.layers.1.post_attention_layernorm.weight": "pytorch_model-00001-of-00006.bin",
24
+ "model.layers.1.self_attn.k_proj.bias": "pytorch_model-00001-of-00006.bin",
25
+ "model.layers.1.self_attn.k_proj.weight": "pytorch_model-00001-of-00006.bin",
26
+ "model.layers.1.self_attn.o_proj.weight": "pytorch_model-00001-of-00006.bin",
27
+ "model.layers.1.self_attn.q_proj.bias": "pytorch_model-00001-of-00006.bin",
28
+ "model.layers.1.self_attn.q_proj.weight": "pytorch_model-00001-of-00006.bin",
29
+ "model.layers.1.self_attn.v_proj.bias": "pytorch_model-00001-of-00006.bin",
30
+ "model.layers.1.self_attn.v_proj.weight": "pytorch_model-00001-of-00006.bin",
31
+ "model.layers.10.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
32
+ "model.layers.10.mlp.down_proj.weight": "pytorch_model-00003-of-00006.bin",
33
+ "model.layers.10.mlp.gate_proj.weight": "pytorch_model-00003-of-00006.bin",
34
+ "model.layers.10.mlp.up_proj.weight": "pytorch_model-00003-of-00006.bin",
35
+ "model.layers.10.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
36
+ "model.layers.10.self_attn.k_proj.bias": "pytorch_model-00003-of-00006.bin",
37
+ "model.layers.10.self_attn.k_proj.weight": "pytorch_model-00003-of-00006.bin",
38
+ "model.layers.10.self_attn.o_proj.weight": "pytorch_model-00003-of-00006.bin",
39
+ "model.layers.10.self_attn.q_proj.bias": "pytorch_model-00003-of-00006.bin",
40
+ "model.layers.10.self_attn.q_proj.weight": "pytorch_model-00003-of-00006.bin",
41
+ "model.layers.10.self_attn.v_proj.bias": "pytorch_model-00003-of-00006.bin",
42
+ "model.layers.10.self_attn.v_proj.weight": "pytorch_model-00003-of-00006.bin",
43
+ "model.layers.11.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
44
+ "model.layers.11.mlp.down_proj.weight": "pytorch_model-00003-of-00006.bin",
45
+ "model.layers.11.mlp.gate_proj.weight": "pytorch_model-00003-of-00006.bin",
46
+ "model.layers.11.mlp.up_proj.weight": "pytorch_model-00003-of-00006.bin",
47
+ "model.layers.11.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
48
+ "model.layers.11.self_attn.k_proj.bias": "pytorch_model-00003-of-00006.bin",
49
+ "model.layers.11.self_attn.k_proj.weight": "pytorch_model-00003-of-00006.bin",
50
+ "model.layers.11.self_attn.o_proj.weight": "pytorch_model-00003-of-00006.bin",
51
+ "model.layers.11.self_attn.q_proj.bias": "pytorch_model-00003-of-00006.bin",
52
+ "model.layers.11.self_attn.q_proj.weight": "pytorch_model-00003-of-00006.bin",
53
+ "model.layers.11.self_attn.v_proj.bias": "pytorch_model-00003-of-00006.bin",
54
+ "model.layers.11.self_attn.v_proj.weight": "pytorch_model-00003-of-00006.bin",
55
+ "model.layers.12.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
56
+ "model.layers.12.mlp.down_proj.weight": "pytorch_model-00003-of-00006.bin",
57
+ "model.layers.12.mlp.gate_proj.weight": "pytorch_model-00003-of-00006.bin",
58
+ "model.layers.12.mlp.up_proj.weight": "pytorch_model-00003-of-00006.bin",
59
+ "model.layers.12.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
60
+ "model.layers.12.self_attn.k_proj.bias": "pytorch_model-00003-of-00006.bin",
61
+ "model.layers.12.self_attn.k_proj.weight": "pytorch_model-00003-of-00006.bin",
62
+ "model.layers.12.self_attn.o_proj.weight": "pytorch_model-00003-of-00006.bin",
63
+ "model.layers.12.self_attn.q_proj.bias": "pytorch_model-00003-of-00006.bin",
64
+ "model.layers.12.self_attn.q_proj.weight": "pytorch_model-00003-of-00006.bin",
65
+ "model.layers.12.self_attn.v_proj.bias": "pytorch_model-00003-of-00006.bin",
66
+ "model.layers.12.self_attn.v_proj.weight": "pytorch_model-00003-of-00006.bin",
67
+ "model.layers.13.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
68
+ "model.layers.13.mlp.down_proj.weight": "pytorch_model-00004-of-00006.bin",
69
+ "model.layers.13.mlp.gate_proj.weight": "pytorch_model-00003-of-00006.bin",
70
+ "model.layers.13.mlp.up_proj.weight": "pytorch_model-00004-of-00006.bin",
71
+ "model.layers.13.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
72
+ "model.layers.13.self_attn.k_proj.bias": "pytorch_model-00003-of-00006.bin",
73
+ "model.layers.13.self_attn.k_proj.weight": "pytorch_model-00003-of-00006.bin",
74
+ "model.layers.13.self_attn.o_proj.weight": "pytorch_model-00003-of-00006.bin",
75
+ "model.layers.13.self_attn.q_proj.bias": "pytorch_model-00003-of-00006.bin",
76
+ "model.layers.13.self_attn.q_proj.weight": "pytorch_model-00003-of-00006.bin",
77
+ "model.layers.13.self_attn.v_proj.bias": "pytorch_model-00003-of-00006.bin",
78
+ "model.layers.13.self_attn.v_proj.weight": "pytorch_model-00003-of-00006.bin",
79
+ "model.layers.14.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
80
+ "model.layers.14.mlp.down_proj.weight": "pytorch_model-00004-of-00006.bin",
81
+ "model.layers.14.mlp.gate_proj.weight": "pytorch_model-00004-of-00006.bin",
82
+ "model.layers.14.mlp.up_proj.weight": "pytorch_model-00004-of-00006.bin",
83
+ "model.layers.14.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
84
+ "model.layers.14.self_attn.k_proj.bias": "pytorch_model-00004-of-00006.bin",
85
+ "model.layers.14.self_attn.k_proj.weight": "pytorch_model-00004-of-00006.bin",
86
+ "model.layers.14.self_attn.o_proj.weight": "pytorch_model-00004-of-00006.bin",
87
+ "model.layers.14.self_attn.q_proj.bias": "pytorch_model-00004-of-00006.bin",
88
+ "model.layers.14.self_attn.q_proj.weight": "pytorch_model-00004-of-00006.bin",
89
+ "model.layers.14.self_attn.v_proj.bias": "pytorch_model-00004-of-00006.bin",
90
+ "model.layers.14.self_attn.v_proj.weight": "pytorch_model-00004-of-00006.bin",
91
+ "model.layers.15.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
92
+ "model.layers.15.mlp.down_proj.weight": "pytorch_model-00004-of-00006.bin",
93
+ "model.layers.15.mlp.gate_proj.weight": "pytorch_model-00004-of-00006.bin",
94
+ "model.layers.15.mlp.up_proj.weight": "pytorch_model-00004-of-00006.bin",
95
+ "model.layers.15.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
96
+ "model.layers.15.self_attn.k_proj.bias": "pytorch_model-00004-of-00006.bin",
97
+ "model.layers.15.self_attn.k_proj.weight": "pytorch_model-00004-of-00006.bin",
98
+ "model.layers.15.self_attn.o_proj.weight": "pytorch_model-00004-of-00006.bin",
99
+ "model.layers.15.self_attn.q_proj.bias": "pytorch_model-00004-of-00006.bin",
100
+ "model.layers.15.self_attn.q_proj.weight": "pytorch_model-00004-of-00006.bin",
101
+ "model.layers.15.self_attn.v_proj.bias": "pytorch_model-00004-of-00006.bin",
102
+ "model.layers.15.self_attn.v_proj.weight": "pytorch_model-00004-of-00006.bin",
103
+ "model.layers.16.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
104
+ "model.layers.16.mlp.down_proj.weight": "pytorch_model-00004-of-00006.bin",
105
+ "model.layers.16.mlp.gate_proj.weight": "pytorch_model-00004-of-00006.bin",
106
+ "model.layers.16.mlp.up_proj.weight": "pytorch_model-00004-of-00006.bin",
107
+ "model.layers.16.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
108
+ "model.layers.16.self_attn.k_proj.bias": "pytorch_model-00004-of-00006.bin",
109
+ "model.layers.16.self_attn.k_proj.weight": "pytorch_model-00004-of-00006.bin",
110
+ "model.layers.16.self_attn.o_proj.weight": "pytorch_model-00004-of-00006.bin",
111
+ "model.layers.16.self_attn.q_proj.bias": "pytorch_model-00004-of-00006.bin",
112
+ "model.layers.16.self_attn.q_proj.weight": "pytorch_model-00004-of-00006.bin",
113
+ "model.layers.16.self_attn.v_proj.bias": "pytorch_model-00004-of-00006.bin",
114
+ "model.layers.16.self_attn.v_proj.weight": "pytorch_model-00004-of-00006.bin",
115
+ "model.layers.17.input_layernorm.weight": "pytorch_model-00004-of-00006.bin",
116
+ "model.layers.17.mlp.down_proj.weight": "pytorch_model-00004-of-00006.bin",
117
+ "model.layers.17.mlp.gate_proj.weight": "pytorch_model-00004-of-00006.bin",
118
+ "model.layers.17.mlp.up_proj.weight": "pytorch_model-00004-of-00006.bin",
119
+ "model.layers.17.post_attention_layernorm.weight": "pytorch_model-00004-of-00006.bin",
120
+ "model.layers.17.self_attn.k_proj.bias": "pytorch_model-00004-of-00006.bin",
121
+ "model.layers.17.self_attn.k_proj.weight": "pytorch_model-00004-of-00006.bin",
122
+ "model.layers.17.self_attn.o_proj.weight": "pytorch_model-00004-of-00006.bin",
123
+ "model.layers.17.self_attn.q_proj.bias": "pytorch_model-00004-of-00006.bin",
124
+ "model.layers.17.self_attn.q_proj.weight": "pytorch_model-00004-of-00006.bin",
125
+ "model.layers.17.self_attn.v_proj.bias": "pytorch_model-00004-of-00006.bin",
126
+ "model.layers.17.self_attn.v_proj.weight": "pytorch_model-00004-of-00006.bin",
127
+ "model.layers.18.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
128
+ "model.layers.18.mlp.down_proj.weight": "pytorch_model-00005-of-00006.bin",
129
+ "model.layers.18.mlp.gate_proj.weight": "pytorch_model-00004-of-00006.bin",
130
+ "model.layers.18.mlp.up_proj.weight": "pytorch_model-00004-of-00006.bin",
131
+ "model.layers.18.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
132
+ "model.layers.18.self_attn.k_proj.bias": "pytorch_model-00004-of-00006.bin",
133
+ "model.layers.18.self_attn.k_proj.weight": "pytorch_model-00004-of-00006.bin",
134
+ "model.layers.18.self_attn.o_proj.weight": "pytorch_model-00004-of-00006.bin",
135
+ "model.layers.18.self_attn.q_proj.bias": "pytorch_model-00004-of-00006.bin",
136
+ "model.layers.18.self_attn.q_proj.weight": "pytorch_model-00004-of-00006.bin",
137
+ "model.layers.18.self_attn.v_proj.bias": "pytorch_model-00004-of-00006.bin",
138
+ "model.layers.18.self_attn.v_proj.weight": "pytorch_model-00004-of-00006.bin",
139
+ "model.layers.19.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
140
+ "model.layers.19.mlp.down_proj.weight": "pytorch_model-00005-of-00006.bin",
141
+ "model.layers.19.mlp.gate_proj.weight": "pytorch_model-00005-of-00006.bin",
142
+ "model.layers.19.mlp.up_proj.weight": "pytorch_model-00005-of-00006.bin",
143
+ "model.layers.19.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
144
+ "model.layers.19.self_attn.k_proj.bias": "pytorch_model-00005-of-00006.bin",
145
+ "model.layers.19.self_attn.k_proj.weight": "pytorch_model-00005-of-00006.bin",
146
+ "model.layers.19.self_attn.o_proj.weight": "pytorch_model-00005-of-00006.bin",
147
+ "model.layers.19.self_attn.q_proj.bias": "pytorch_model-00005-of-00006.bin",
148
+ "model.layers.19.self_attn.q_proj.weight": "pytorch_model-00005-of-00006.bin",
149
+ "model.layers.19.self_attn.v_proj.bias": "pytorch_model-00005-of-00006.bin",
150
+ "model.layers.19.self_attn.v_proj.weight": "pytorch_model-00005-of-00006.bin",
151
+ "model.layers.2.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
152
+ "model.layers.2.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
153
+ "model.layers.2.mlp.gate_proj.weight": "pytorch_model-00001-of-00006.bin",
154
+ "model.layers.2.mlp.up_proj.weight": "pytorch_model-00001-of-00006.bin",
155
+ "model.layers.2.post_attention_layernorm.weight": "pytorch_model-00001-of-00006.bin",
156
+ "model.layers.2.self_attn.k_proj.bias": "pytorch_model-00001-of-00006.bin",
157
+ "model.layers.2.self_attn.k_proj.weight": "pytorch_model-00001-of-00006.bin",
158
+ "model.layers.2.self_attn.o_proj.weight": "pytorch_model-00001-of-00006.bin",
159
+ "model.layers.2.self_attn.q_proj.bias": "pytorch_model-00001-of-00006.bin",
160
+ "model.layers.2.self_attn.q_proj.weight": "pytorch_model-00001-of-00006.bin",
161
+ "model.layers.2.self_attn.v_proj.bias": "pytorch_model-00001-of-00006.bin",
162
+ "model.layers.2.self_attn.v_proj.weight": "pytorch_model-00001-of-00006.bin",
163
+ "model.layers.20.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
164
+ "model.layers.20.mlp.down_proj.weight": "pytorch_model-00005-of-00006.bin",
165
+ "model.layers.20.mlp.gate_proj.weight": "pytorch_model-00005-of-00006.bin",
166
+ "model.layers.20.mlp.up_proj.weight": "pytorch_model-00005-of-00006.bin",
167
+ "model.layers.20.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
168
+ "model.layers.20.self_attn.k_proj.bias": "pytorch_model-00005-of-00006.bin",
169
+ "model.layers.20.self_attn.k_proj.weight": "pytorch_model-00005-of-00006.bin",
170
+ "model.layers.20.self_attn.o_proj.weight": "pytorch_model-00005-of-00006.bin",
171
+ "model.layers.20.self_attn.q_proj.bias": "pytorch_model-00005-of-00006.bin",
172
+ "model.layers.20.self_attn.q_proj.weight": "pytorch_model-00005-of-00006.bin",
173
+ "model.layers.20.self_attn.v_proj.bias": "pytorch_model-00005-of-00006.bin",
174
+ "model.layers.20.self_attn.v_proj.weight": "pytorch_model-00005-of-00006.bin",
175
+ "model.layers.21.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
176
+ "model.layers.21.mlp.down_proj.weight": "pytorch_model-00005-of-00006.bin",
177
+ "model.layers.21.mlp.gate_proj.weight": "pytorch_model-00005-of-00006.bin",
178
+ "model.layers.21.mlp.up_proj.weight": "pytorch_model-00005-of-00006.bin",
179
+ "model.layers.21.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
180
+ "model.layers.21.self_attn.k_proj.bias": "pytorch_model-00005-of-00006.bin",
181
+ "model.layers.21.self_attn.k_proj.weight": "pytorch_model-00005-of-00006.bin",
182
+ "model.layers.21.self_attn.o_proj.weight": "pytorch_model-00005-of-00006.bin",
183
+ "model.layers.21.self_attn.q_proj.bias": "pytorch_model-00005-of-00006.bin",
184
+ "model.layers.21.self_attn.q_proj.weight": "pytorch_model-00005-of-00006.bin",
185
+ "model.layers.21.self_attn.v_proj.bias": "pytorch_model-00005-of-00006.bin",
186
+ "model.layers.21.self_attn.v_proj.weight": "pytorch_model-00005-of-00006.bin",
187
+ "model.layers.22.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
188
+ "model.layers.22.mlp.down_proj.weight": "pytorch_model-00005-of-00006.bin",
189
+ "model.layers.22.mlp.gate_proj.weight": "pytorch_model-00005-of-00006.bin",
190
+ "model.layers.22.mlp.up_proj.weight": "pytorch_model-00005-of-00006.bin",
191
+ "model.layers.22.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
192
+ "model.layers.22.self_attn.k_proj.bias": "pytorch_model-00005-of-00006.bin",
193
+ "model.layers.22.self_attn.k_proj.weight": "pytorch_model-00005-of-00006.bin",
194
+ "model.layers.22.self_attn.o_proj.weight": "pytorch_model-00005-of-00006.bin",
195
+ "model.layers.22.self_attn.q_proj.bias": "pytorch_model-00005-of-00006.bin",
196
+ "model.layers.22.self_attn.q_proj.weight": "pytorch_model-00005-of-00006.bin",
197
+ "model.layers.22.self_attn.v_proj.bias": "pytorch_model-00005-of-00006.bin",
198
+ "model.layers.22.self_attn.v_proj.weight": "pytorch_model-00005-of-00006.bin",
199
+ "model.layers.23.input_layernorm.weight": "pytorch_model-00005-of-00006.bin",
200
+ "model.layers.23.mlp.down_proj.weight": "pytorch_model-00005-of-00006.bin",
201
+ "model.layers.23.mlp.gate_proj.weight": "pytorch_model-00005-of-00006.bin",
202
+ "model.layers.23.mlp.up_proj.weight": "pytorch_model-00005-of-00006.bin",
203
+ "model.layers.23.post_attention_layernorm.weight": "pytorch_model-00005-of-00006.bin",
204
+ "model.layers.23.self_attn.k_proj.bias": "pytorch_model-00005-of-00006.bin",
205
+ "model.layers.23.self_attn.k_proj.weight": "pytorch_model-00005-of-00006.bin",
206
+ "model.layers.23.self_attn.o_proj.weight": "pytorch_model-00005-of-00006.bin",
207
+ "model.layers.23.self_attn.q_proj.bias": "pytorch_model-00005-of-00006.bin",
208
+ "model.layers.23.self_attn.q_proj.weight": "pytorch_model-00005-of-00006.bin",
209
+ "model.layers.23.self_attn.v_proj.bias": "pytorch_model-00005-of-00006.bin",
210
+ "model.layers.23.self_attn.v_proj.weight": "pytorch_model-00005-of-00006.bin",
211
+ "model.layers.24.input_layernorm.weight": "pytorch_model-00006-of-00006.bin",
212
+ "model.layers.24.mlp.down_proj.weight": "pytorch_model-00006-of-00006.bin",
213
+ "model.layers.24.mlp.gate_proj.weight": "pytorch_model-00006-of-00006.bin",
214
+ "model.layers.24.mlp.up_proj.weight": "pytorch_model-00006-of-00006.bin",
215
+ "model.layers.24.post_attention_layernorm.weight": "pytorch_model-00006-of-00006.bin",
216
+ "model.layers.24.self_attn.k_proj.bias": "pytorch_model-00005-of-00006.bin",
217
+ "model.layers.24.self_attn.k_proj.weight": "pytorch_model-00005-of-00006.bin",
218
+ "model.layers.24.self_attn.o_proj.weight": "pytorch_model-00006-of-00006.bin",
219
+ "model.layers.24.self_attn.q_proj.bias": "pytorch_model-00005-of-00006.bin",
220
+ "model.layers.24.self_attn.q_proj.weight": "pytorch_model-00005-of-00006.bin",
221
+ "model.layers.24.self_attn.v_proj.bias": "pytorch_model-00005-of-00006.bin",
222
+ "model.layers.24.self_attn.v_proj.weight": "pytorch_model-00005-of-00006.bin",
223
+ "model.layers.25.input_layernorm.weight": "pytorch_model-00006-of-00006.bin",
224
+ "model.layers.25.mlp.down_proj.weight": "pytorch_model-00006-of-00006.bin",
225
+ "model.layers.25.mlp.gate_proj.weight": "pytorch_model-00006-of-00006.bin",
226
+ "model.layers.25.mlp.up_proj.weight": "pytorch_model-00006-of-00006.bin",
227
+ "model.layers.25.post_attention_layernorm.weight": "pytorch_model-00006-of-00006.bin",
228
+ "model.layers.25.self_attn.k_proj.bias": "pytorch_model-00006-of-00006.bin",
229
+ "model.layers.25.self_attn.k_proj.weight": "pytorch_model-00006-of-00006.bin",
230
+ "model.layers.25.self_attn.o_proj.weight": "pytorch_model-00006-of-00006.bin",
231
+ "model.layers.25.self_attn.q_proj.bias": "pytorch_model-00006-of-00006.bin",
232
+ "model.layers.25.self_attn.q_proj.weight": "pytorch_model-00006-of-00006.bin",
233
+ "model.layers.25.self_attn.v_proj.bias": "pytorch_model-00006-of-00006.bin",
234
+ "model.layers.25.self_attn.v_proj.weight": "pytorch_model-00006-of-00006.bin",
235
+ "model.layers.26.input_layernorm.weight": "pytorch_model-00006-of-00006.bin",
236
+ "model.layers.26.mlp.down_proj.weight": "pytorch_model-00006-of-00006.bin",
237
+ "model.layers.26.mlp.gate_proj.weight": "pytorch_model-00006-of-00006.bin",
238
+ "model.layers.26.mlp.up_proj.weight": "pytorch_model-00006-of-00006.bin",
239
+ "model.layers.26.post_attention_layernorm.weight": "pytorch_model-00006-of-00006.bin",
240
+ "model.layers.26.self_attn.k_proj.bias": "pytorch_model-00006-of-00006.bin",
241
+ "model.layers.26.self_attn.k_proj.weight": "pytorch_model-00006-of-00006.bin",
242
+ "model.layers.26.self_attn.o_proj.weight": "pytorch_model-00006-of-00006.bin",
243
+ "model.layers.26.self_attn.q_proj.bias": "pytorch_model-00006-of-00006.bin",
244
+ "model.layers.26.self_attn.q_proj.weight": "pytorch_model-00006-of-00006.bin",
245
+ "model.layers.26.self_attn.v_proj.bias": "pytorch_model-00006-of-00006.bin",
246
+ "model.layers.26.self_attn.v_proj.weight": "pytorch_model-00006-of-00006.bin",
247
+ "model.layers.27.input_layernorm.weight": "pytorch_model-00006-of-00006.bin",
248
+ "model.layers.27.mlp.down_proj.weight": "pytorch_model-00006-of-00006.bin",
249
+ "model.layers.27.mlp.gate_proj.weight": "pytorch_model-00006-of-00006.bin",
250
+ "model.layers.27.mlp.up_proj.weight": "pytorch_model-00006-of-00006.bin",
251
+ "model.layers.27.post_attention_layernorm.weight": "pytorch_model-00006-of-00006.bin",
252
+ "model.layers.27.self_attn.k_proj.bias": "pytorch_model-00006-of-00006.bin",
253
+ "model.layers.27.self_attn.k_proj.weight": "pytorch_model-00006-of-00006.bin",
254
+ "model.layers.27.self_attn.o_proj.weight": "pytorch_model-00006-of-00006.bin",
255
+ "model.layers.27.self_attn.q_proj.bias": "pytorch_model-00006-of-00006.bin",
256
+ "model.layers.27.self_attn.q_proj.weight": "pytorch_model-00006-of-00006.bin",
257
+ "model.layers.27.self_attn.v_proj.bias": "pytorch_model-00006-of-00006.bin",
258
+ "model.layers.27.self_attn.v_proj.weight": "pytorch_model-00006-of-00006.bin",
259
+ "model.layers.3.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
260
+ "model.layers.3.mlp.down_proj.weight": "pytorch_model-00002-of-00006.bin",
261
+ "model.layers.3.mlp.gate_proj.weight": "pytorch_model-00002-of-00006.bin",
262
+ "model.layers.3.mlp.up_proj.weight": "pytorch_model-00002-of-00006.bin",
263
+ "model.layers.3.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
264
+ "model.layers.3.self_attn.k_proj.bias": "pytorch_model-00002-of-00006.bin",
265
+ "model.layers.3.self_attn.k_proj.weight": "pytorch_model-00002-of-00006.bin",
266
+ "model.layers.3.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
267
+ "model.layers.3.self_attn.q_proj.bias": "pytorch_model-00002-of-00006.bin",
268
+ "model.layers.3.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
269
+ "model.layers.3.self_attn.v_proj.bias": "pytorch_model-00002-of-00006.bin",
270
+ "model.layers.3.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
271
+ "model.layers.4.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
272
+ "model.layers.4.mlp.down_proj.weight": "pytorch_model-00002-of-00006.bin",
273
+ "model.layers.4.mlp.gate_proj.weight": "pytorch_model-00002-of-00006.bin",
274
+ "model.layers.4.mlp.up_proj.weight": "pytorch_model-00002-of-00006.bin",
275
+ "model.layers.4.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
276
+ "model.layers.4.self_attn.k_proj.bias": "pytorch_model-00002-of-00006.bin",
277
+ "model.layers.4.self_attn.k_proj.weight": "pytorch_model-00002-of-00006.bin",
278
+ "model.layers.4.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
279
+ "model.layers.4.self_attn.q_proj.bias": "pytorch_model-00002-of-00006.bin",
280
+ "model.layers.4.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
281
+ "model.layers.4.self_attn.v_proj.bias": "pytorch_model-00002-of-00006.bin",
282
+ "model.layers.4.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
283
+ "model.layers.5.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
284
+ "model.layers.5.mlp.down_proj.weight": "pytorch_model-00002-of-00006.bin",
285
+ "model.layers.5.mlp.gate_proj.weight": "pytorch_model-00002-of-00006.bin",
286
+ "model.layers.5.mlp.up_proj.weight": "pytorch_model-00002-of-00006.bin",
287
+ "model.layers.5.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
288
+ "model.layers.5.self_attn.k_proj.bias": "pytorch_model-00002-of-00006.bin",
289
+ "model.layers.5.self_attn.k_proj.weight": "pytorch_model-00002-of-00006.bin",
290
+ "model.layers.5.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
291
+ "model.layers.5.self_attn.q_proj.bias": "pytorch_model-00002-of-00006.bin",
292
+ "model.layers.5.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
293
+ "model.layers.5.self_attn.v_proj.bias": "pytorch_model-00002-of-00006.bin",
294
+ "model.layers.5.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
295
+ "model.layers.6.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
296
+ "model.layers.6.mlp.down_proj.weight": "pytorch_model-00002-of-00006.bin",
297
+ "model.layers.6.mlp.gate_proj.weight": "pytorch_model-00002-of-00006.bin",
298
+ "model.layers.6.mlp.up_proj.weight": "pytorch_model-00002-of-00006.bin",
299
+ "model.layers.6.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
300
+ "model.layers.6.self_attn.k_proj.bias": "pytorch_model-00002-of-00006.bin",
301
+ "model.layers.6.self_attn.k_proj.weight": "pytorch_model-00002-of-00006.bin",
302
+ "model.layers.6.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
303
+ "model.layers.6.self_attn.q_proj.bias": "pytorch_model-00002-of-00006.bin",
304
+ "model.layers.6.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
305
+ "model.layers.6.self_attn.v_proj.bias": "pytorch_model-00002-of-00006.bin",
306
+ "model.layers.6.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
307
+ "model.layers.7.input_layernorm.weight": "pytorch_model-00002-of-00006.bin",
308
+ "model.layers.7.mlp.down_proj.weight": "pytorch_model-00002-of-00006.bin",
309
+ "model.layers.7.mlp.gate_proj.weight": "pytorch_model-00002-of-00006.bin",
310
+ "model.layers.7.mlp.up_proj.weight": "pytorch_model-00002-of-00006.bin",
311
+ "model.layers.7.post_attention_layernorm.weight": "pytorch_model-00002-of-00006.bin",
312
+ "model.layers.7.self_attn.k_proj.bias": "pytorch_model-00002-of-00006.bin",
313
+ "model.layers.7.self_attn.k_proj.weight": "pytorch_model-00002-of-00006.bin",
314
+ "model.layers.7.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
315
+ "model.layers.7.self_attn.q_proj.bias": "pytorch_model-00002-of-00006.bin",
316
+ "model.layers.7.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
317
+ "model.layers.7.self_attn.v_proj.bias": "pytorch_model-00002-of-00006.bin",
318
+ "model.layers.7.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
319
+ "model.layers.8.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
320
+ "model.layers.8.mlp.down_proj.weight": "pytorch_model-00003-of-00006.bin",
321
+ "model.layers.8.mlp.gate_proj.weight": "pytorch_model-00003-of-00006.bin",
322
+ "model.layers.8.mlp.up_proj.weight": "pytorch_model-00003-of-00006.bin",
323
+ "model.layers.8.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
324
+ "model.layers.8.self_attn.k_proj.bias": "pytorch_model-00002-of-00006.bin",
325
+ "model.layers.8.self_attn.k_proj.weight": "pytorch_model-00002-of-00006.bin",
326
+ "model.layers.8.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
327
+ "model.layers.8.self_attn.q_proj.bias": "pytorch_model-00002-of-00006.bin",
328
+ "model.layers.8.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
329
+ "model.layers.8.self_attn.v_proj.bias": "pytorch_model-00002-of-00006.bin",
330
+ "model.layers.8.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
331
+ "model.layers.9.input_layernorm.weight": "pytorch_model-00003-of-00006.bin",
332
+ "model.layers.9.mlp.down_proj.weight": "pytorch_model-00003-of-00006.bin",
333
+ "model.layers.9.mlp.gate_proj.weight": "pytorch_model-00003-of-00006.bin",
334
+ "model.layers.9.mlp.up_proj.weight": "pytorch_model-00003-of-00006.bin",
335
+ "model.layers.9.post_attention_layernorm.weight": "pytorch_model-00003-of-00006.bin",
336
+ "model.layers.9.self_attn.k_proj.bias": "pytorch_model-00003-of-00006.bin",
337
+ "model.layers.9.self_attn.k_proj.weight": "pytorch_model-00003-of-00006.bin",
338
+ "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00003-of-00006.bin",
339
+ "model.layers.9.self_attn.q_proj.bias": "pytorch_model-00003-of-00006.bin",
340
+ "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00003-of-00006.bin",
341
+ "model.layers.9.self_attn.v_proj.bias": "pytorch_model-00003-of-00006.bin",
342
+ "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00003-of-00006.bin",
343
+ "model.norm.weight": "pytorch_model-00006-of-00006.bin",
344
+ "score.bias": "pytorch_model-00006-of-00006.bin",
345
+ "score.weight": "pytorch_model-00006-of-00006.bin"
346
+ }
347
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "<|im_start|>",
4
+ "<|im_end|>"
5
+ ],
6
+ "eos_token": {
7
+ "content": "<|endoftext|>",
8
+ "lstrip": false,
9
+ "normalized": false,
10
+ "rstrip": false,
11
+ "single_word": false
12
+ },
13
+ "pad_token": {
14
+ "content": "<|endoftext|>",
15
+ "lstrip": false,
16
+ "normalized": false,
17
+ "rstrip": false,
18
+ "single_word": false
19
+ }
20
+ }
tokenizer_config.json ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "added_tokens_decoder": {
4
+ "151643": {
5
+ "content": "<|endoftext|>",
6
+ "lstrip": false,
7
+ "normalized": false,
8
+ "rstrip": false,
9
+ "single_word": false,
10
+ "special": true
11
+ },
12
+ "151644": {
13
+ "content": "<|im_start|>",
14
+ "lstrip": false,
15
+ "normalized": false,
16
+ "rstrip": false,
17
+ "single_word": false,
18
+ "special": true
19
+ },
20
+ "151645": {
21
+ "content": "<|im_end|>",
22
+ "lstrip": false,
23
+ "normalized": false,
24
+ "rstrip": false,
25
+ "single_word": false,
26
+ "special": true
27
+ }
28
+ },
29
+ "additional_special_tokens": [
30
+ "<|im_start|>",
31
+ "<|im_end|>"
32
+ ],
33
+ "bos_token": null,
34
+ "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{{ system_message }}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ content }}{% elif message['role'] == 'assistant' %}{{ content }}{% endif %}{% endfor %}",
35
+ "clean_up_tokenization_spaces": false,
36
+ "eos_token": "<|endoftext|>",
37
+ "errors": "replace",
38
+ "model_max_length": 32768,
39
+ "pad_token": "<|endoftext|>",
40
+ "padding_side": "right",
41
+ "split_special_tokens": false,
42
+ "tokenizer_class": "Qwen2Tokenizer",
43
+ "unk_token": null
44
+ }
upos.py ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from transformers import TokenClassificationPipeline
2
+
3
+ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
4
+ def __init__(self,**kwargs):
5
+ import numpy
6
+ super().__init__(**kwargs)
7
+ x=self.model.config.label2id
8
+ y=[k for k in x if not k.startswith("I-")]
9
+ self.transition=numpy.full((len(x),len(x)),numpy.nan)
10
+ for k,v in x.items():
11
+ for j in ["I-"+k[2:]] if k.startswith("B-") else [k]+y if k.startswith("I-") else y:
12
+ self.transition[v,x[j]]=0
13
+ def check_model_type(self,supported_models):
14
+ pass
15
+ def postprocess(self,model_outputs,**kwargs):
16
+ import numpy
17
+ if "logits" not in model_outputs:
18
+ return self.postprocess(model_outputs[0],**kwargs)
19
+ m=model_outputs["logits"][0].numpy()
20
+ e=numpy.exp(m-numpy.max(m,axis=-1,keepdims=True))
21
+ z=e/e.sum(axis=-1,keepdims=True)
22
+ for i in range(m.shape[0]-1,0,-1):
23
+ m[i-1]+=numpy.nanmax(m[i]+self.transition,axis=1)
24
+ k=[numpy.nanargmax(m[0]+self.transition[0])]
25
+ for i in range(1,m.shape[0]):
26
+ k.append(numpy.nanargmax(m[i]+self.transition[k[-1]]))
27
+ w=[{"entity":self.model.config.id2label[j],"start":s,"end":e,"score":z[i,j]} for i,((s,e),j) in enumerate(zip(model_outputs["offset_mapping"][0].tolist(),k)) if s<e]
28
+ if "aggregation_strategy" in kwargs and kwargs["aggregation_strategy"]!="none":
29
+ for i,t in reversed(list(enumerate(w))):
30
+ p=t.pop("entity")
31
+ if p.startswith("I-"):
32
+ w[i-1]["score"]=min(w[i-1]["score"],t["score"])
33
+ w[i-1]["end"]=w.pop(i)["end"]
34
+ elif p.startswith("B-"):
35
+ t["entity_group"]=p[2:]
36
+ else:
37
+ t["entity_group"]=p
38
+ for t in w:
39
+ t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
40
+ return w
41
+
vocab.json ADDED
The diff for this file is too large to render. See raw diff