model improved for transformers 4.42

Browse files

Files changed (12) hide show

config.json +1 -12
maker.sh +6 -38
pytorch_model-00001-of-00006.bin +1 -1
pytorch_model-00002-of-00006.bin +1 -1
pytorch_model-00003-of-00006.bin +1 -1
pytorch_model-00004-of-00006.bin +1 -1
pytorch_model-00005-of-00006.bin +1 -1
pytorch_model-00006-of-00006.bin +1 -1
pytorch_model.bin.index.json +3 -3
tokenizer.json +575 -582
tokenizer_config.json +1 -0
upos.py +1 -40

config.json CHANGED Viewed

@@ -4,22 +4,11 @@
   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.LlamaForTokenClassification"
-  },
   "bos_token_id": 1,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,
@@ -166,7 +155,7 @@
   "tie_word_embeddings": false,
   "tokenizer_class": "LlamaTokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.41.2",
   "use_cache": true,
   "vocab_size": 43744
 }

   ],
   "attention_bias": false,
   "attention_dropout": 0.0,
   "bos_token_id": 1,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,
   "tie_word_embeddings": false,
   "tokenizer_class": "LlamaTokenizerFast",
   "torch_dtype": "float32",
+  "transformers_version": "4.42.4",
   "use_cache": true,
   "vocab_size": 43744
 }

maker.sh CHANGED Viewed

@@ -1,49 +1,17 @@
 #! /bin/sh
 test -f ja_gsd_modern.conllu || curl -LO https://github.com/KoichiYasuoka/SuPar-UniDic/raw/main/suparunidic/suparmodels/ja_gsd_modern.conllu
-curl -L https://huggingface.co/KoichiYasuoka/Swallow-7b-plus-upos/resolve/main/tokenizer.json | env LANG=ja_JP.utf8 egrep -v '"[ぁ-ん] [ぁ-ん]",$' > newtokenizer.json
 TMP=./maker$$.py
 cat << 'EOF' > $TMP
 #! /usr/bin/env deepspeed
 src="KoichiYasuoka/Swallow-7b-plus-upos"
 tgt="KoichiYasuoka/Swallow-7b-plus-char-upos"
-from transformers import LlamaTokenizerFast,LlamaModel,LlamaPreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
-class LlamaForTokenClassification(LlamaPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=LlamaModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[1:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):

 #! /bin/sh
 test -f ja_gsd_modern.conllu || curl -LO https://github.com/KoichiYasuoka/SuPar-UniDic/raw/main/suparunidic/suparmodels/ja_gsd_modern.conllu
+( if [ -f KoichiYasuoka/Swallow-7b-plus-upos/tokenizer.json ]
+  then cat KoichiYasuoka/Swallow-7b-plus-upos/tokenizer.json
+  else curl -L https://huggingface.co/KoichiYasuoka/Swallow-7b-plus-upos/resolve/main/tokenizer.json
+  fi
+) | env LANG=ja_JP.utf8 egrep -v '"[ぁ-ん] [ぁ-ん]",$' > newtokenizer.json
 TMP=./maker$$.py
 cat << 'EOF' > $TMP
 #! /usr/bin/env deepspeed
 src="KoichiYasuoka/Swallow-7b-plus-upos"
 tgt="KoichiYasuoka/Swallow-7b-plus-char-upos"
+from transformers import LlamaTokenizerFast,LlamaForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):

pytorch_model-00001-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:b13ae6b0fd6bf2779fb4e4df457e3c693b91369869b0fff330130c4f99b2c20e
 size 4965712452

 version https://git-lfs.github.com/spec/v1
+oid sha256:692bdb261ef8779cd58b516450b18d082d7094f1ac2ba46db95454f8c4197da5
 size 4965712452

pytorch_model-00002-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d896a30f76e9b1fee9f183ad8638a6cedc1da563f3d2d01278a7dc9b46a3d32a
 size 4924328556

 version https://git-lfs.github.com/spec/v1
+oid sha256:09d0e2bcffb1ce5e9cd95ea6977d474eb8d2c7eebdbfe1ce8f802c1db0096d65
 size 4924328556

pytorch_model-00003-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fdf4712dba3e338ead5f8c8c0559890d2e0fe568da1dfd1d0f58b62c9dbf97a1
 size 4857219294

 version https://git-lfs.github.com/spec/v1
+oid sha256:d25051601bcf446f572c009773090c033538dd7b810beb52271e957494ba562f
 size 4857219294

pytorch_model-00004-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7aabf1ca7f7963a9b2901bff5c30ccf6454e91ac374f7fed7fd6e443ddb069d7
 size 4857219294

 version https://git-lfs.github.com/spec/v1
+oid sha256:15db31b8c730a7df1599e3121316de388024aee68ae6d543092e93a130dcb555
 size 4857219294

pytorch_model-00005-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fc2f6bb378f573913af9194eca2339fb5077cfe849f9ed9e8f36fa46877b3ba1
 size 4857219294

 version https://git-lfs.github.com/spec/v1
+oid sha256:989af73b70f3c362fdda3f67e5044496814128c3d133f8872977dd07d13c48a6
 size 4857219294

pytorch_model-00006-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9aa4184bf98e7e4595cc57088ac56eef0943927147a29ae4b60c14559d7090e3
 size 2161173694

 version https://git-lfs.github.com/spec/v1
+oid sha256:e8483e76dc3e91fed1ab66073cbcd439d62342414eee644239b38c637e7bec8b
 size 2161173694

pytorch_model.bin.index.json CHANGED Viewed

@@ -3,8 +3,6 @@
     "total_size": 26622771440
   },
   "weight_map": {
-    "classifier.bias": "pytorch_model-00006-of-00006.bin",
-    "classifier.weight": "pytorch_model-00006-of-00006.bin",
     "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
@@ -294,6 +292,8 @@
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
-    "model.norm.weight": "pytorch_model-00006-of-00006.bin"
   }
 }

     "total_size": 26622771440
   },
   "weight_map": {
     "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
+    "model.norm.weight": "pytorch_model-00006-of-00006.bin",
+    "score.bias": "pytorch_model-00006-of-00006.bin",
+    "score.weight": "pytorch_model-00006-of-00006.bin"
   }
 }

tokenizer.json CHANGED Viewed

@@ -31,14 +31,13 @@
       "special": true
     }
   ],
-  "normalizer": {
-    "type": "Replace",
-    "pattern": {
-      "String": " "
-    },
-    "content": "▁"
   },
-  "pre_tokenizer": null,
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
@@ -108,12 +107,6 @@
       },
       {
         "type": "Fuse"
-      },
-      {
-        "type": "Strip",
-        "content": " ",
-        "start": 1,
-        "stop": 0
       }
     ]
   },
@@ -124,7 +117,7 @@
     "continuing_subword_prefix": null,
     "end_of_word_suffix": null,
     "fuse_unk": true,
-    "byte_fallback": false,
     "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
@@ -43303,574 +43296,574 @@
       "趙": 43173,
       "弛": 43174,
       "徊": 43175,
-      "搆": 43176,
-      "槓": 43177,
-      "淇": 43178,
-      "呵": 43179,
-      "糀": 43180,
-      "遁": 43181,
-      "鄙": 43182,
-      "寶": 43183,
-      "禎": 43184,
-      "儕": 43185,
-      "嚼": 43186,
-      "僞": 43187,
-      "吏": 43188,
-      "勒": 43189,
-      "饗": 43190,
-      "舒": 43191,
-      "梃": 43192,
-      "朧": 43193,
-      "殷": 43194,
-      "爲": 43195,
-      "巍": 43196,
-      "專": 43197,
-      "拮": 43198,
-      "貽": 43199,
-      "勸": 43200,
-      "毆": 43201,
-      "竭": 43202,
-      "賣": 43203,
-      "汀": 43204,
-      "豕": 43205,
-      "籾": 43206,
-      "恤": 43207,
-      "觀": 43208,
-      "麾": 43209,
-      "跋": 43210,
-      "勵": 43211,
-      "毘": 43212,
-      "證": 43213,
-      "壓": 43214,
-      "彗": 43215,
-      "倹": 43216,
-      "諛": 43217,
-      "狡": 43218,
-      "裴": 43219,
-      "覽": 43220,
-      "燮": 43221,
-      "黨": 43222,
-      "咀": 43223,
-      "濤": 43224,
-      "埴": 43225,
-      "亥": 43226,
-      "熾": 43227,
-      "袈": 43228,
-      "尤": 43229,
-      "讓": 43230,
-      "惟": 43231,
-      "駭": 43232,
-      "價": 43233,
-      "咎": 43234,
-      "駢": 43235,
-      "魏": 43236,
-      "酉": 43237,
-      "錮": 43238,
-      "唖": 43239,
-      "饜": 43240,
-      "况": 43241,
-      "恪": 43242,
-      "墮": 43243,
-      "戾": 43244,
-      "縣": 43245,
-      "諮": 43246,
-      "攝": 43247,
-      "俄": 43248,
-      "屬": 43249,
-      "翹": 43250,
-      "瑣": 43251,
-      "蛾": 43252,
-      "闍": 43253,
-      "勅": 43254,
-      "驅": 43255,
-      "詔": 43256,
-      "囮": 43257,
-      "拔": 43258,
-      "悖": 43259,
-      "衷": 43260,
-      "膽": 43261,
-      "碩": 43262,
-      "濟": 43263,
-      "逕": 43264,
-      "羸": 43265,
-      "浹": 43266,
-      "覊": 43267,
-      "諂": 43268,
-      "隱": 43269,
-      "傅": 43270,
-      "醨": 43271,
-      "餘": 43272,
-      "頗": 43273,
-      "舍": 43274,
-      "緡": 43275,
-      "滊": 43276,
-      "菑": 43277,
-      "媢": 43278,
-      "壯": 43279,
-      "游": 43280,
-      "藥": 43281,
-      "楔": 43282,
-      "讀": 43283,
-      "內": 43284,
-      "麿": 43285,
-      "亂": 43286,
-      "蘊": 43287,
-      "廢": 43288,
-      "籌": 43289,
-      "竝": 43290,
-      "蓉": 43291,
-      "憚": 43292,
-      "孛": 43293,
-      "澳": 43294,
-      "恂": 43295,
-      "擧": 43296,
-      "鵠": 43297,
-      "纉": 43298,
-      "帥": 43299,
-      "嘲": 43300,
-      "揠": 43301,
-      "肅": 43302,
-      "僩": 43303,
-      "斤": 43304,
-      "埠": 43305,
-      "靜": 43306,
-      "朕": 43307,
-      "乎": 43308,
-      "禽": 43309,
-      "庸": 43310,
-      "黽": 43311,
-      "纂": 43312,
-      "僨": 43313,
-      "騁": 43314,
-      "羹": 43315,
-      "𠮟": 43316,
-      "欷": 43317,
-      "皈": 43318,
-      "鋒": 43319,
-      "對": 43320,
-      "歟": 43321,
-      "雜": 43322,
-      "醉": 43323,
-      "虧": 43324,
-      "沮": 43325,
-      "榮": 43326,
-      "僥": 43327,
-      "傀": 43328,
-      "姦": 43329,
-      "駸": 43330,
-      "稱": 43331,
-      "薛": 43332,
-      "龕": 43333,
-      "嬰": 43334,
-      "攬": 43335,
-      "掩": 43336,
-      "聚": 43337,
-      "閻": 43338,
-      "舅": 43339,
-      "輕": 43340,
-      "綮": 43341,
-      "變": 43342,
-      "飮": 43343,
-      "匈": 43344,
-      "蓁": 43345,
-      "岨": 43346,
-      "貶": 43347,
-      "畧": 43348,
-      "胥": 43349,
-      "闢": 43350,
-      "厘": 43351,
-      "隋": 43352,
-      "聲": 43353,
-      "塡": 43354,
-      "弐": 43355,
-      "勞": 43356,
-      "參": 43357,
-      "濶": 43358,
-      "罷": 43359,
-      "穆": 43360,
-      "允": 43361,
-      "榜": 43362,
-      "蜻": 43363,
-      "貮": 43364,
-      "詭": 43365,
-      "憾": 43366,
-      "伍": 43367,
-      "隧": 43368,
-      "擕": 43369,
-      "諱": 43370,
-      "汝": 43371,
-      "傳": 43372,
-      "錐": 43373,
-      "舷": 43374,
-      "裨": 43375,
-      "冰": 43376,
-      "營": 43377,
-      "聘": 43378,
-      "欣": 43379,
-      "饒": 43380,
-      "恆": 43381,
-      "僻": 43382,
-      "駑": 43383,
-      "滌": 43384,
-      "舊": 43385,
-      "鵬": 43386,
-      "肚": 43387,
-      "遑": 43388,
-      "淺": 43389,
-      "彥": 43390,
-      "臾": 43391,
-      "獻": 43392,
-      "丙": 43393,
-      "發": 43394,
-      "皺": 43395,
-      "墾": 43396,
-      "俟": 43397,
-      "嚮": 43398,
-      "擔": 43399,
-      "畫": 43400,
-      "誥": 43401,
-      "諺": 43402,
-      "驕": 43403,
-      "甍": 43404,
-      "沃": 43405,
-      "謨": 43406,
-      "蹄": 43407,
-      "黠": 43408,
-      "闊": 43409,
-      "關": 43410,
-      "痍": 43411,
-      "悌": 43412,
-      "繼": 43413,
-      "篩": 43414,
-      "體": 43415,
-      "鸞": 43416,
-      "畢": 43417,
-      "恰": 43418,
-      "頒": 43419,
-      "毋": 43420,
-      "歸": 43421,
-      "决": 43422,
-      "蠻": 43423,
-      "愼": 43424,
-      "橙": 43425,
-      "賈": 43426,
-      "迭": 43427,
-      "悉": 43428,
-      "辟": 43429,
-      "壹": 43430,
-      "脹": 43431,
-      "淫": 43432,
-      "芒": 43433,
-      "絜": 43434,
-      "竊": 43435,
-      "猗": 43436,
-      "諠": 43437,
-      "實": 43438,
-      "舜": 43439,
-      "釀": 43440,
-      "盖": 43441,
-      "揆": 43442,
-      "擇": 43443,
-      "摧": 43444,
-      "尸": 43445,
-      "巖": 43446,
-      "罕": 43447,
-      "膠": 43448,
-      "臘": 43449,
-      "愈": 43450,
-      "躓": 43451,
-      "逓": 43452,
-      "盈": 43453,
-      "璽": 43454,
-      "峻": 43455,
-      "鐸": 43456,
-      "翰": 43457,
-      "謁": 43458,
-      "揖": 43459,
-      "忒": 43460,
-      "賂": 43461,
-      "趨": 43462,
-      "蔗": 43463,
-      "從": 43464,
-      "鎭": 43465,
-      "煥": 43466,
-      "宋": 43467,
-      "乘": 43468,
-      "裔": 43469,
-      "雖": 43470,
-      "桀": 43471,
-      "胖": 43472,
-      "渺": 43473,
-      "扈": 43474,
-      "儁": 43475,
-      "攘": 43476,
-      "斥": 43477,
-      "耘": 43478,
-      "缺": 43479,
-      "沒": 43480,
-      "當": 43481,
-      "諟": 43482,
-      "羣": 43483,
-      "憮": 43484,
-      "畏": 43485,
-      "處": 43486,
-      "厭": 43487,
-      "禀": 43488,
-      "倖": 43489,
-      "蜀": 43490,
-      "鏃": 43491,
-      "傚": 43492,
-      "誣": 43493,
-      "騷": 43494,
-      "譬": 43495,
-      "惠": 43496,
-      "烟": 43497,
-      "爰": 43498,
-      "滿": 43499,
-      "險": 43500,
-      "廠": 43501,
-      "憬": 43502,
-      "嗚": 43503,
-      "甞": 43504,
-      "崗": 43505,
-      "玻": 43506,
-      "盜": 43507,
-      "獸": 43508,
-      "愧": 43509,
-      "碇": 43510,
-      "憊": 43511,
-      "隨": 43512,
-      "魯": 43513,
-      "亞": 43514,
-      "孟": 43515,
-      "耻": 43516,
-      "雉": 43517,
-      "緝": 43518,
-      "埸": 43519,
-      "闡": 43520,
-      "殉": 43521,
-      "丞": 43522,
-      "瀑": 43523,
-      "轉": 43524,
-      "與": 43525,
-      "諧": 43526,
-      "艱": 43527,
-      "澁": 43528,
-      "糢": 43529,
-      "辨": 43530,
-      "權": 43531,
-      "抔": 43532,
-      "尹": 43533,
-      "哨": 43534,
-      "矩": 43535,
-      "樣": 43536,
-      "筵": 43537,
-      "豈": 43538,
-      "辭": 43539,
-      "戊": 43540,
-      "熙": 43541,
-      "恙": 43542,
-      "宍": 43543,
-      "痘": 43544,
-      "斂": 43545,
-      "陷": 43546,
-      "縱": 43547,
-      "臀": 43548,
-      "倩": 43549,
-      "圓": 43550,
-      "醵": 43551,
-      "菉": 43552,
-      "懥": 43553,
-      "葢": 43554,
-      "儼": 43555,
-      "陋": 43556,
-      "柘": 43557,
-      "脩": 43558,
-      "窩": 43559,
-      "吻": 43560,
-      "豫": 43561,
-      "茲": 43562,
-      "糾": 43563,
-      "僇": 43564,
-      "犂": 43565,
-      "咤": 43566,
-      "逈": 43567,
-      "袁": 43568,
-      "據": 43569,
-      "飜": 43570,
-      "狸": 43571,
-      "酋": 43572,
-      "撹": 43573,
-      "亨": 43574,
-      "觸": 43575,
-      "礦": 43576,
-      "繭": 43577,
-      "堡": 43578,
-      "陪": 43579,
-      "婿": 43580,
-      "篆": 43581,
-      "驗": 43582,
-      "皷": 43583,
-      "慘": 43584,
-      "圖": 43585,
-      "假": 43586,
-      "棍": 43587,
-      "襄": 43588,
-      "戮": 43589,
-      "耆": 43590,
-      "廟": 43591,
-      "佩": 43592,
-      "蘐": 43593,
-      "恕": 43594,
-      "鑒": 43595,
-      "儘": 43596,
-      "鳶": 43597,
-      "恣": 43598,
-      "鬢": 43599,
-      "盡": 43600,
-      "堯": 43601,
-      "諷": 43602,
-      "疆": 43603,
-      "續": 43604,
-      "劾": 43605,
-      "歡": 43606,
-      "稍": 43607,
-      "嫡": 43608,
-      "顯": 43609,
-      "亦": 43610,
-      "猾": 43611,
-      "竢": 43612,
-      "聊": 43613,
-      "蛉": 43614,
-      "珊": 43615,
-      "虔": 43616,
-      "臂": 43617,
-      "臥": 43618,
-      "戲": 43619,
-      "逬": 43620,
-      "匐": 43621,
-      "寵": 43622,
-      "蹊": 43623,
-      "歎": 43624,
-      "蕃": 43625,
-      "簪": 43626,
-      "挈": 43627,
-      "斷": 43628,
-      "筭": 43629,
-      "獨": 43630,
-      "盍": 43631,
-      "尙": 43632,
-      "賤": 43633,
-      "虞": 43634,
-      "竟": 43635,
-      "贖": 43636,
-      "巓": 43637,
-      "屡": 43638,
-      "聽": 43639,
-      "團": 43640,
-      "蛯": 43641,
-      "竄": 43642,
-      "將": 43643,
-      "轍": 43644,
-      "扣": 43645,
-      "鰭": 43646,
-      "頽": 43647,
-      "懼": 43648,
-      "碍": 43649,
-      "數": 43650,
-      "馭": 43651,
-      "搖": 43652,
-      "惡": 43653,
-      "啻": 43654,
-      "綬": 43655,
-      "禮": 43656,
-      "紂": 43657,
-      "懺": 43658,
-      "儒": 43659,
-      "楯": 43660,
-      "覺": 43661,
-      "諫": 43662,
-      "拷": 43663,
-      "謭": 43664,
-      "爨": 43665,
-      "蹈": 43666,
-      "碎": 43667,
-      "嚴": 43668,
-      "誦": 43669,
-      "辜": 43670,
-      "單": 43671,
-      "鉤": 43672,
-      "剝": 43673,
-      "兒": 43674,
-      "宥": 43675,
-      "毫": 43676,
-      "槙": 43677,
-      "擾": 43678,
-      "楷": 43679,
-      "慄": 43680,
-      "寔": 43681,
-      "茫": 43682,
-      "壤": 43683,
-      "菰": 43684,
-      "忿": 43685,
-      "爭": 43686,
-      "遲": 43687,
-      "彭": 43688,
-      "譯": 43689,
-      "雞": 43690,
-      "殲": 43691,
-      "瑟": 43692,
-      "頴": 43693,
-      "聯": 43694,
-      "禊": 43695,
-      "窘": 43696,
-      "恊": 43697,
-      "醗": 43698,
-      "奧": 43699,
-      "皓": 43700,
-      "胤": 43701,
-      "儡": 43702,
-      "葦": 43703,
-      "靡": 43704,
-      "瞻": 43705,
-      "赫": 43706,
-      "苟": 43707,
-      "灼": 43708,
-      "歐": 43709,
-      "揜": 43710,
-      "敖": 43711,
-      "苅": 43712,
-      "寃": 43713,
-      "殘": 43714,
-      "巌": 43715,
-      "兩": 43716,
-      "洽": 43717,
-      "撿": 43718,
-      "總": 43719,
-      "夭": 43720,
-      "邑": 43721,
-      "岌": 43722,
-      "歔": 43723,
-      "徽": 43724,
-      "鐵": 43725,
-      "檣": 43726,
-      "肆": 43727,
-      "號": 43728,
-      "羞": 43729,
-      "靱": 43730,
-      "逼": 43731,
-      "鄭": 43732,
-      "啜": 43733,
-      "頰": 43734,
-      "戌": 43735,
-      "矜": 43736,
-      "鍾": 43737,
-      "妾": 43738,
-      "彝": 43739,
-      "欵": 43740,
-      "漸": 43741,
-      "敎": 43742,
-      "匍": 43743
     },
     "merges": [
       "▁ t",

       "special": true
     }
   ],
+  "normalizer": null,
+  "pre_tokenizer": {
+    "type": "Metaspace",
+    "replacement": "▁",
+    "prepend_scheme": "never",
+    "split": false
   },
   "post_processor": {
     "type": "TemplateProcessing",
     "single": [
       },
       {
         "type": "Fuse"
       }
     ]
   },
     "continuing_subword_prefix": null,
     "end_of_word_suffix": null,
     "fuse_unk": true,
+    "byte_fallback": true,
     "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "趙": 43173,
       "弛": 43174,
       "徊": 43175,
+      "逕": 43176,
+      "撹": 43177,
+      "碎": 43178,
+      "逼": 43179,
+      "鳶": 43180,
+      "毘": 43181,
+      "參": 43182,
+      "攬": 43183,
+      "夭": 43184,
+      "罷": 43185,
+      "贖": 43186,
+      "劾": 43187,
+      "儁": 43188,
+      "禎": 43189,
+      "闍": 43190,
+      "覺": 43191,
+      "汝": 43192,
+      "畧": 43193,
+      "儒": 43194,
+      "埸": 43195,
+      "藥": 43196,
+      "愼": 43197,
+      "賣": 43198,
+      "剝": 43199,
+      "斂": 43200,
+      "緝": 43201,
+      "恆": 43202,
+      "遑": 43203,
+      "號": 43204,
+      "咎": 43205,
+      "譯": 43206,
+      "堯": 43207,
+      "葢": 43208,
+      "錮": 43209,
+      "飮": 43210,
+      "匍": 43211,
+      "漸": 43212,
+      "歔": 43213,
+      "貽": 43214,
+      "巌": 43215,
+      "嚮": 43216,
+      "陪": 43217,
+      "專": 43218,
+      "壓": 43219,
+      "膠": 43220,
+      "瀑": 43221,
+      "尹": 43222,
+      "險": 43223,
+      "鐵": 43224,
+      "濟": 43225,
+      "單": 43226,
+      "滿": 43227,
+      "竭": 43228,
+      "沃": 43229,
+      "綮": 43230,
+      "顯": 43231,
+      "澁": 43232,
+      "榜": 43233,
+      "麾": 43234,
+      "蛾": 43235,
+      "欣": 43236,
+      "袁": 43237,
+      "諧": 43238,
+      "扈": 43239,
+      "駑": 43240,
+      "罕": 43241,
+      "媢": 43242,
+      "艱": 43243,
+      "疆": 43244,
+      "樣": 43245,
+      "歟": 43246,
+      "儘": 43247,
+      "宋": 43248,
+      "賂": 43249,
+      "洽": 43250,
+      "虧": 43251,
+      "薛": 43252,
+      "耆": 43253,
+      "駭": 43254,
+      "權": 43255,
+      "慘": 43256,
+      "賈": 43257,
+      "鄙": 43258,
+      "矜": 43259,
+      "汀": 43260,
+      "羹": 43261,
+      "敎": 43262,
+      "肚": 43263,
+      "揠": 43264,
+      "鐸": 43265,
+      "蘐": 43266,
+      "奧": 43267,
+      "畫": 43268,
+      "吏": 43269,
+      "竟": 43270,
+      "茲": 43271,
+      "壹": 43272,
+      "瞻": 43273,
+      "靜": 43274,
+      "翹": 43275,
+      "饒": 43276,
+      "廠": 43277,
+      "絜": 43278,
+      "脩": 43279,
+      "雉": 43280,
+      "膽": 43281,
+      "蓉": 43282,
+      "靱": 43283,
+      "蹈": 43284,
+      "總": 43285,
+      "蠻": 43286,
+      "陋": 43287,
+      "囮": 43288,
+      "從": 43289,
+      "恕": 43290,
+      "實": 43291,
+      "內": 43292,
+      "盖": 43293,
+      "竄": 43294,
+      "畏": 43295,
+      "雖": 43296,
+      "瑣": 43297,
+      "俟": 43298,
+      "獻": 43299,
+      "愧": 43300,
+      "餘": 43301,
+      "畢": 43302,
+      "耘": 43303,
+      "恙": 43304,
+      "謨": 43305,
+      "赫": 43306,
+      "鉤": 43307,
+      "舍": 43308,
+      "謁": 43309,
+      "戊": 43310,
+      "鬢": 43311,
+      "蜀": 43312,
+      "婿": 43313,
+      "儕": 43314,
+      "淇": 43315,
+      "巓": 43316,
+      "恤": 43317,
+      "團": 43318,
+      "愈": 43319,
+      "丙": 43320,
+      "鏃": 43321,
+      "簪": 43322,
+      "僩": 43323,
+      "彗": 43324,
+      "讀": 43325,
+      "豈": 43326,
+      "誥": 43327,
+      "僞": 43328,
+      "礦": 43329,
+      "勒": 43330,
+      "苅": 43331,
+      "發": 43332,
+      "闢": 43333,
+      "鄭": 43334,
+      "酉": 43335,
+      "僻": 43336,
+      "倩": 43337,
+      "驗": 43338,
+      "胖": 43339,
+      "岨": 43340,
+      "兒": 43341,
+      "蜻": 43342,
+      "舜": 43343,
+      "搆": 43344,
+      "圓": 43345,
+      "勸": 43346,
+      "尸": 43347,
+      "盡": 43348,
+      "恣": 43349,
+      "盜": 43350,
+      "聊": 43351,
+      "烟": 43352,
+      "勵": 43353,
+      "臘": 43354,
+      "竝": 43355,
+      "諺": 43356,
+      "戌": 43357,
+      "碍": 43358,
+      "亦": 43359,
+      "勞": 43360,
+      "賤": 43361,
+      "殘": 43362,
+      "舒": 43363,
+      "緡": 43364,
+      "糾": 43365,
+      "擾": 43366,
+      "穆": 43367,
+      "崗": 43368,
+      "寵": 43369,
+      "謭": 43370,
+      "儡": 43371,
+      "嬰": 43372,
+      "騷": 43373,
+      "戾": 43374,
+      "籾": 43375,
+      "輕": 43376,
+      "圖": 43377,
+      "黨": 43378,
+      "棍": 43379,
+      "殉": 43380,
+      "僇": 43381,
+      "淺": 43382,
+      "璽": 43383,
+      "與": 43384,
+      "煥": 43385,
+      "摧": 43386,
+      "痘": 43387,
+      "欷": 43388,
+      "鰭": 43389,
+      "淫": 43390,
+      "跋": 43391,
+      "襄": 43392,
+      "酋": 43393,
+      "碩": 43394,
+      "苟": 43395,
+      "梃": 43396,
+      "關": 43397,
+      "狸": 43398,
+      "饜": 43399,
+      "稍": 43400,
+      "燮": 43401,
+      "歡": 43402,
+      "辨": 43403,
+      "孛": 43404,
+      "堡": 43405,
+      "楷": 43406,
+      "擕": 43407,
+      "諂": 43408,
+      "傅": 43409,
+      "倖": 43410,
+      "灼": 43411,
+      "迭": 43412,
+      "詔": 43413,
+      "殷": 43414,
+      "呵": 43415,
+      "忿": 43416,
+      "傀": 43417,
+      "墾": 43418,
+      "巖": 43419,
+      "滊": 43420,
+      "俄": 43421,
+      "竢": 43422,
+      "歎": 43423,
+      "啻": 43424,
+      "庸": 43425,
+      "糢": 43426,
+      "鍾": 43427,
+      "爨": 43428,
+      "拮": 43429,
+      "揆": 43430,
+      "乎": 43431,
+      "槙": 43432,
+      "逈": 43433,
+      "騁": 43434,
+      "裴": 43435,
+      "屡": 43436,
+      "筵": 43437,
+      "趨": 43438,
+      "皈": 43439,
+      "徽": 43440,
+      "蛉": 43441,
+      "頗": 43442,
+      "蕃": 43443,
+      "寔": 43444,
+      "憚": 43445,
+      "辟": 43446,
+      "龕": 43447,
+      "臾": 43448,
+      "壤": 43449,
+      "舅": 43450,
+      "黽": 43451,
+      "恂": 43452,
+      "帥": 43453,
+      "舷": 43454,
+      "擔": 43455,
+      "攝": 43456,
+      "廟": 43457,
+      "袈": 43458,
+      "醵": 43459,
+      "駸": 43460,
+      "聯": 43461,
+      "鸞": 43462,
+      "頽": 43463,
+      "脹": 43464,
+      "醗": 43465,
+      "禀": 43466,
+      "戲": 43467,
+      "恪": 43468,
+      "覽": 43469,
+      "懼": 43470,
+      "隋": 43471,
+      "遁": 43472,
+      "爭": 43473,
+      "瑟": 43474,
+      "宍": 43475,
+      "峻": 43476,
+      "殲": 43477,
+      "黠": 43478,
+      "厭": 43479,
+      "歸": 43480,
+      "蛯": 43481,
+      "憬": 43482,
+      "營": 43483,
+      "朧": 43484,
+      "塡": 43485,
+      "當": 43486,
+      "抔": 43487,
+      "缺": 43488,
+      "尤": 43489,
+      "熙": 43490,
+      "爰": 43491,
+      "肅": 43492,
+      "禽": 43493,
+      "懺": 43494,
+      "豫": 43495,
+      "馭": 43496,
+      "猾": 43497,
+      "廢": 43498,
+      "躓": 43499,
+      "甍": 43500,
+      "虞": 43501,
+      "邑": 43502,
+      "窘": 43503,
+      "葦": 43504,
+      "驅": 43505,
+      "羣": 43506,
+      "闊": 43507,
+      "惠": 43508,
+      "芒": 43509,
+      "屬": 43510,
+      "岌": 43511,
+      "魯": 43512,
+      "錐": 43513,
+      "醨": 43514,
+      "朕": 43515,
+      "乘": 43516,
+      "濤": 43517,
+      "矩": 43518,
+      "隱": 43519,
+      "嘲": 43520,
+      "欵": 43521,
+      "柘": 43522,
+      "熾": 43523,
+      "醉": 43524,
+      "伍": 43525,
+      "紂": 43526,
+      "覊": 43527,
+      "麿": 43528,
+      "毫": 43529,
+      "寶": 43530,
+      "裨": 43531,
+      "亨": 43532,
+      "轉": 43533,
+      "盍": 43534,
+      "碇": 43535,
+      "皺": 43536,
+      "唖": 43537,
+      "閻": 43538,
+      "蓁": 43539,
+      "虔": 43540,
+      "丞": 43541,
+      "橙": 43542,
+      "變": 43543,
+      "臥": 43544,
+      "渺": 43545,
+      "蘊": 43546,
+      "恰": 43547,
+      "纉": 43548,
+      "亂": 43549,
+      "諱": 43550,
+      "亞": 43551,
+      "啜": 43552,
+      "桀": 43553,
+      "諟": 43554,
+      "楔": 43555,
+      "恊": 43556,
+      "冰": 43557,
+      "斥": 43558,
+      "驕": 43559,
+      "儼": 43560,
+      "體": 43561,
+      "挈": 43562,
+      "駢": 43563,
+      "皓": 43564,
+      "誣": 43565,
+      "頒": 43566,
+      "鎭": 43567,
+      "蹄": 43568,
+      "續": 43569,
+      "狡": 43570,
+      "魏": 43571,
+      "觀": 43572,
+      "沮": 43573,
+      "逓": 43574,
+      "詭": 43575,
+      "懥": 43576,
+      "處": 43577,
+      "盈": 43578,
+      "窩": 43579,
+      "匐": 43580,
+      "慄": 43581,
+      "遲": 43582,
+      "傚": 43583,
+      "搖": 43584,
+      "鵬": 43585,
+      "惟": 43586,
+      "菑": 43587,
+      "綬": 43588,
+      "臀": 43589,
+      "籌": 43590,
+      "憮": 43591,
+      "貮": 43592,
+      "縱": 43593,
+      "聚": 43594,
+      "釀": 43595,
+      "轍": 43596,
+      "兩": 43597,
+      "嚼": 43598,
+      "豕": 43599,
+      "闡": 43600,
+      "禮": 43601,
+      "戮": 43602,
+      "埠": 43603,
+      "雜": 43604,
+      "拔": 43605,
+      "吻": 43606,
+      "獨": 43607,
+      "扣": 43608,
+      "將": 43609,
+      "聽": 43610,
+      "諫": 43611,
+      "諠": 43612,
+      "繭": 43613,
+      "尙": 43614,
+      "彭": 43615,
+      "决": 43616,
+      "咤": 43617,
+      "飜": 43618,
+      "證": 43619,
+      "繼": 43620,
+      "嗚": 43621,
+      "鑒": 43622,
+      "彝": 43623,
+      "歐": 43624,
+      "玻": 43625,
+      "僥": 43626,
+      "諷": 43627,
+      "僨": 43628,
+      "浹": 43629,
+      "禊": 43630,
+      "允": 43631,
+      "𠮟": 43632,
+      "撿": 43633,
+      "犂": 43634,
+      "毆": 43635,
+      "筭": 43636,
+      "陷": 43637,
+      "澳": 43638,
+      "惡": 43639,
+      "纂": 43640,
+      "衷": 43641,
+      "諛": 43642,
+      "楯": 43643,
+      "蹊": 43644,
+      "隨": 43645,
+      "亥": 43646,
+      "弐": 43647,
+      "嫡": 43648,
+      "誦": 43649,
+      "寃": 43650,
+      "巍": 43651,
+      "宥": 43652,
+      "悉": 43653,
+      "猗": 43654,
+      "聘": 43655,
+      "壯": 43656,
+      "憊": 43657,
+      "掩": 43658,
+      "隧": 43659,
+      "胤": 43660,
+      "羞": 43661,
+      "爲": 43662,
+      "羸": 43663,
+      "擇": 43664,
+      "觸": 43665,
+      "糀": 43666,
+      "翰": 43667,
+      "游": 43668,
+      "辜": 43669,
+      "忒": 43670,
+      "厘": 43671,
+      "數": 43672,
+      "濶": 43673,
+      "傳": 43674,
+      "鵠": 43675,
+      "篆": 43676,
+      "譬": 43677,
+      "哨": 43678,
+      "斷": 43679,
+      "嚴": 43680,
+      "肆": 43681,
+      "珊": 43682,
+      "悌": 43683,
+      "孟": 43684,
+      "稱": 43685,
+      "沒": 43686,
+      "墮": 43687,
+      "獸": 43688,
+      "妾": 43689,
+      "揖": 43690,
+      "擧": 43691,
+      "悖": 43692,
+      "况": 43693,
+      "篩": 43694,
+      "滌": 43695,
+      "勅": 43696,
+      "姦": 43697,
+      "舊": 43698,
+      "痍": 43699,
+      "頰": 43700,
+      "聲": 43701,
+      "裔": 43702,
+      "毋": 43703,
+      "諮": 43704,
+      "咀": 43705,
+      "耻": 43706,
+      "對": 43707,
+      "蔗": 43708,
+      "假": 43709,
+      "彥": 43710,
+      "雞": 43711,
+      "拷": 43712,
+      "揜": 43713,
+      "甞": 43714,
+      "逬": 43715,
+      "榮": 43716,
+      "讓": 43717,
+      "竊": 43718,
+      "辭": 43719,
+      "斤": 43720,
+      "檣": 43721,
+      "茫": 43722,
+      "憾": 43723,
+      "埴": 43724,
+      "鋒": 43725,
+      "饗": 43726,
+      "敖": 43727,
+      "靡": 43728,
+      "倹": 43729,
+      "槓": 43730,
+      "價": 43731,
+      "攘": 43732,
+      "胥": 43733,
+      "臂": 43734,
+      "菉": 43735,
+      "菰": 43736,
+      "佩": 43737,
+      "匈": 43738,
+      "貶": 43739,
+      "縣": 43740,
+      "據": 43741,
+      "皷": 43742,
+      "頴": 43743
     },
     "merges": [
       "▁ t",

tokenizer_config.json CHANGED Viewed

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",

 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",

upos.py CHANGED Viewed

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,LlamaModel,LlamaPreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -40,41 +39,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-class LlamaForTokenClassification(LlamaPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=LlamaModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)

+from transformers import TokenClassificationPipeline
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w