model improved for transformers 4.42

Browse files

Files changed (12) hide show

config.json +1 -15
maker.sh +3 -39
pytorch_model-00001-of-00006.bin +2 -2
pytorch_model-00002-of-00006.bin +2 -2
pytorch_model-00003-of-00006.bin +1 -1
pytorch_model-00004-of-00006.bin +2 -2
pytorch_model-00005-of-00006.bin +1 -1
pytorch_model-00006-of-00006.bin +1 -1
pytorch_model.bin.index.json +3 -3
tokenizer.json +517 -516
tokenizer_config.json +2 -0
upos.py +1 -40

config.json CHANGED Viewed

@@ -2,24 +2,12 @@
   "architectures": [
     "MistralForTokenClassification"
   ],
-  "attention_bias": false,
   "attention_dropout": 0.0,
-  "auto_map": {
-    "AutoModelForTokenClassification": "upos.MistralForTokenClassification"
-  },
   "bos_token_id": 1,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
-    },
-    "token-classification": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
-    },
-    "ner": {
-      "impl": "upos.RawTokenClassificationPipeline",
-      "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,
@@ -156,15 +144,13 @@
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
-  "pretraining_tp": 1,
   "rms_norm_eps": 1e-05,
-  "rope_scaling": null,
   "rope_theta": 10000.0,
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "tokenizer_class": "LlamaTokenizerFast",
   "torch_dtype": "float32",
-  "transformers_version": "4.38.1",
   "use_cache": true,
   "vocab_size": 43317
 }

   "architectures": [
     "MistralForTokenClassification"
   ],
   "attention_dropout": 0.0,
   "bos_token_id": 1,
   "custom_pipelines": {
     "upos": {
       "impl": "upos.BellmanFordTokenClassificationPipeline",
       "pt": "AutoModelForTokenClassification"
     }
   },
   "eos_token_id": 2,
   "num_attention_heads": 32,
   "num_hidden_layers": 32,
   "num_key_value_heads": 8,
   "rms_norm_eps": 1e-05,
   "rope_theta": 10000.0,
   "sliding_window": 4096,
   "tie_word_embeddings": false,
   "tokenizer_class": "LlamaTokenizerFast",
   "torch_dtype": "float32",
+  "transformers_version": "4.42.4",
   "use_cache": true,
   "vocab_size": 43317
 }

maker.sh CHANGED Viewed

@@ -9,7 +9,7 @@ then TMPA=./maker$$a.py
 src="tokyotech-llm/Swallow-MS-7b-v0.1"
 tgt="exSwallow-MS-7b-v0.1"
 import json,torch,unicodedata
-from transformers import LlamaTokenizerFast,LlamaForCausalLM
 with open("JapaneseCoreKanji.txt","r",encoding="utf-8") as r:
   cjk=[chr(int(t,16)) for t in r.read().strip().split("\n") if not t.startswith("#")]
 with open("ja_gsd_modern.conllu","r",encoding="utf-8") as r:
@@ -26,7 +26,7 @@ d=json.loads(tkz.backend_tokenizer.to_str())
 for i,j in enumerate(c,len(tkz)):
   d["model"]["vocab"][j]=i
 tkz.backend_tokenizer.from_str(json.dumps(d)).save("tokenizer.json")
-mdl=LlamaForCausalLM.from_pretrained(src)
 tkz=LlamaTokenizerFast(tokenizer_file="tokenizer.json",model_max_length=mdl.config.max_position_embeddings,cls_token="<s>",sep_token="<s>",mask_token="<unk>",pad_token="</s>")
 e=mdl.resize_token_embeddings(len(tkz))
 f=mdl.get_output_embeddings()
@@ -48,45 +48,9 @@ cat << 'EOF' > $TMPB
 #! /usr/bin/env deepspeed
 src="exSwallow-MS-7b-v0.1"
 tgt="KoichiYasuoka/Swallow-MS-7b-upos"
-from transformers import LlamaTokenizerFast,MistralModel,MistralPreTrainedModel,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
-from transformers.modeling_outputs import TokenClassifierOutput
 from tokenizers.normalizers import Replace
-class MistralForTokenClassification(MistralPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=MistralModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):
     self.conllu=open(conllu,"r",encoding="utf-8")

 src="tokyotech-llm/Swallow-MS-7b-v0.1"
 tgt="exSwallow-MS-7b-v0.1"
 import json,torch,unicodedata
+from transformers import LlamaTokenizerFast,MistralForCausalLM
 with open("JapaneseCoreKanji.txt","r",encoding="utf-8") as r:
   cjk=[chr(int(t,16)) for t in r.read().strip().split("\n") if not t.startswith("#")]
 with open("ja_gsd_modern.conllu","r",encoding="utf-8") as r:
 for i,j in enumerate(c,len(tkz)):
   d["model"]["vocab"][j]=i
 tkz.backend_tokenizer.from_str(json.dumps(d)).save("tokenizer.json")
+mdl=MistralForCausalLM.from_pretrained(src)
 tkz=LlamaTokenizerFast(tokenizer_file="tokenizer.json",model_max_length=mdl.config.max_position_embeddings,cls_token="<s>",sep_token="<s>",mask_token="<unk>",pad_token="</s>")
 e=mdl.resize_token_embeddings(len(tkz))
 f=mdl.get_output_embeddings()
 #! /usr/bin/env deepspeed
 src="exSwallow-MS-7b-v0.1"
 tgt="KoichiYasuoka/Swallow-MS-7b-upos"
+from transformers import LlamaTokenizerFast,MistralForTokenClassification,AutoConfig,DataCollatorForTokenClassification,TrainingArguments,Trainer
 from tokenizers.normalizers import Replace
 class UPOSFileDataset(object):
   def __init__(self,conllu,tokenizer):
     self.conllu=open(conllu,"r",encoding="utf-8")

pytorch_model-00001-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e6d84221d64e2e200ded9278ecfde0568bae10c9fb37ce332d0204da2f2bd05c
-size 4837046848

 version https://git-lfs.github.com/spec/v1
+oid sha256:3bc383e2836bfade4a66c8c3c8e5871240d5b1cc86aaf912120c098dc37db88f
+size 2539520

pytorch_model-00002-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9dff65e498bee680f1c713066816815201df578e91f946dff3b2916d4a06c710
-size 4999825256

 version https://git-lfs.github.com/spec/v1
+oid sha256:b2a123e02228dbfea708d4ff57dc9ccf7bfd5dbb992567a86d42ab14146b55d2
+size 4390338560

pytorch_model-00003-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:09bcc41cb1dc25584295cdd197a59ebeeb28d82742df8e60fc8441ab82c9c8c0
 size 4999825316

 version https://git-lfs.github.com/spec/v1
+oid sha256:70f9676f885d30e08789261553c6ed7a949acef9de9a67dce005cede7ec8fa73
 size 4999825316

pytorch_model-00004-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:abcd008f22397fe546185c4a3f47601e693e9d2dfc015146f93a546f965e41a0
-size 4832018324

 version https://git-lfs.github.com/spec/v1
+oid sha256:7b8a16ad523ebf2fa508012f2b0d0fee6d2c2a4583daff8d4058aaf7240cc020
+size 23379968

pytorch_model-00005-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9f5a5f0b7a8bd6b3f2ad8792fef499b63ae99f1cb3209029ff07f20b9d6f4c47
 size 4999825320

 version https://git-lfs.github.com/spec/v1
+oid sha256:2ba5e8032cd9a92cfaf0f3d30360285d03a7f32af32b0ad07f0c0b7d241d64f5
 size 4999825320

pytorch_model-00006-of-00006.bin CHANGED Viewed

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e39aca1f76ff251d2945a1b8966204baa98480df9ec4edd7da2a115da8314b3f
 size 3960601264

 version https://git-lfs.github.com/spec/v1
+oid sha256:4a82d77158930bdf9ffc566bd2be9e7600b4b4831d287679e3086cab45466ac6
 size 3960601264

pytorch_model.bin.index.json CHANGED Viewed

@@ -3,8 +3,6 @@
     "total_size": 28629041392
   },
   "weight_map": {
-    "classifier.bias": "pytorch_model-00006-of-00006.bin",
-    "classifier.weight": "pytorch_model-00006-of-00006.bin",
     "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
@@ -294,6 +292,8 @@
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
-    "model.norm.weight": "pytorch_model-00006-of-00006.bin"
   }
 }

     "total_size": 28629041392
   },
   "weight_map": {
     "model.embed_tokens.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.input_layernorm.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.0.mlp.down_proj.weight": "pytorch_model-00001-of-00006.bin",
     "model.layers.9.self_attn.o_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.q_proj.weight": "pytorch_model-00002-of-00006.bin",
     "model.layers.9.self_attn.v_proj.weight": "pytorch_model-00002-of-00006.bin",
+    "model.norm.weight": "pytorch_model-00006-of-00006.bin",
+    "score.bias": "pytorch_model-00006-of-00006.bin",
+    "score.weight": "pytorch_model-00006-of-00006.bin"
   }
 }

tokenizer.json CHANGED Viewed

@@ -125,6 +125,7 @@
     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,
@@ -42926,523 +42927,523 @@
       "勅": 42797,
       "婿": 42798,
       "魯": 42799,
-      "頰": 42800,
-      "戌": 42801,
-      "諮": 42802,
-      "覽": 42803,
-      "虧": 42804,
-      "芒": 42805,
-      "廟": 42806,
-      "雜": 42807,
-      "撿": 42808,
-      "皓": 42809,
-      "恊": 42810,
-      "埸": 42811,
-      "斥": 42812,
-      "辜": 42813,
-      "嘲": 42814,
-      "嫡": 42815,
-      "謁": 42816,
-      "彭": 42817,
-      "耻": 42818,
-      "毋": 42819,
-      "嬰": 42820,
-      "厭": 42821,
-      "帥": 42822,
-      "辨": 42823,
-      "絜": 42824,
-      "鄭": 42825,
-      "諺": 42826,
-      "鏃": 42827,
-      "舊": 42828,
-      "擔": 42829,
-      "誦": 42830,
-      "彝": 42831,
-      "墮": 42832,
-      "嚼": 42833,
-      "鉤": 42834,
-      "錐": 42835,
-      "匐": 42836,
-      "俟": 42837,
-      "憬": 42838,
-      "釀": 42839,
-      "盡": 42840,
-      "諱": 42841,
-      "糢": 42842,
-      "僻": 42843,
-      "瑣": 42844,
-      "逼": 42845,
-      "犂": 42846,
-      "闍": 42847,
-      "覺": 42848,
-      "虔": 42849,
-      "譬": 42850,
-      "禎": 42851,
-      "蛾": 42852,
-      "胖": 42853,
-      "燮": 42854,
-      "躓": 42855,
-      "懼": 42856,
-      "獨": 42857,
-      "圓": 42858,
-      "逓": 42859,
-      "筵": 42860,
-      "瞻": 42861,
-      "騷": 42862,
-      "愧": 42863,
-      "攬": 42864,
-      "廠": 42865,
-      "槓": 42866,
-      "賣": 42867,
-      "勵": 42868,
-      "擕": 42869,
-      "蜀": 42870,
-      "紂": 42871,
-      "甞": 42872,
-      "楔": 42873,
-      "獻": 42874,
-      "巖": 42875,
-      "斂": 42876,
-      "舷": 42877,
-      "諫": 42878,
-      "挈": 42879,
-      "羣": 42880,
-      "茫": 42881,
-      "痘": 42882,
-      "猾": 42883,
-      "彥": 42884,
-      "狡": 42885,
-      "尸": 42886,
-      "憾": 42887,
-      "嚮": 42888,
-      "淫": 42889,
-      "歎": 42890,
-      "驗": 42891,
-      "鸞": 42892,
-      "飜": 42893,
-      "檣": 42894,
-      "戾": 42895,
-      "僞": 42896,
-      "滌": 42897,
-      "胤": 42898,
-      "悌": 42899,
-      "赫": 42900,
-      "濟": 42901,
-      "憚": 42902,
-      "諷": 42903,
-      "黨": 42904,
-      "篆": 42905,
-      "籾": 42906,
-      "靜": 42907,
-      "竢": 42908,
-      "團": 42909,
-      "險": 42910,
-      "儒": 42911,
-      "澁": 42912,
-      "巍": 42913,
-      "膠": 42914,
-      "淇": 42915,
-      "夭": 42916,
-      "頴": 42917,
-      "豫": 42918,
-      "浹": 42919,
-      "惡": 42920,
-      "澳": 42921,
-      "歡": 42922,
-      "賤": 42923,
-      "沃": 42924,
-      "吏": 42925,
-      "臂": 42926,
-      "毘": 42927,
-      "慄": 42928,
-      "竄": 42929,
-      "靡": 42930,
-      "搆": 42931,
-      "𠮟": 42932,
-      "兩": 42933,
-      "斷": 42934,
-      "醨": 42935,
-      "糾": 42936,
-      "聘": 42937,
-      "蛯": 42938,
-      "劾": 42939,
-      "巌": 42940,
-      "謭": 42941,
-      "傅": 42942,
-      "匈": 42943,
-      "臾": 42944,
-      "兒": 42945,
-      "皷": 42946,
-      "梃": 42947,
-      "遁": 42948,
-      "崗": 42949,
-      "諟": 42950,
-      "爲": 42951,
-      "甍": 42952,
-      "駸": 42953,
-      "飮": 42954,
-      "虞": 42955,
-      "殘": 42956,
-      "纂": 42957,
-      "處": 42958,
-      "埠": 42959,
-      "禀": 42960,
-      "馭": 42961,
-      "佩": 42962,
-      "揠": 42963,
-      "邑": 42964,
-      "丙": 42965,
-      "尙": 42966,
-      "鑒": 42967,
-      "彗": 42968,
-      "愈": 42969,
-      "繭": 42970,
-      "鍾": 42971,
-      "碍": 42972,
-      "驕": 42973,
-      "碩": 42974,
-      "狸": 42975,
-      "攝": 42976,
-      "歔": 42977,
-      "鳶": 42978,
-      "讀": 42979,
-      "峻": 42980,
-      "薛": 42981,
-      "盈": 42982,
-      "懺": 42983,
-      "忿": 42984,
-      "鵬": 42985,
-      "饜": 42986,
-      "珊": 42987,
-      "揜": 42988,
-      "輕": 42989,
-      "脩": 42990,
-      "獸": 42991,
-      "朧": 42992,
-      "鐸": 42993,
-      "恤": 42994,
-      "丞": 42995,
-      "諧": 42996,
-      "皺": 42997,
-      "專": 42998,
-      "屡": 42999,
-      "諠": 43000,
-      "拔": 43001,
-      "閻": 43002,
-      "皈": 43003,
-      "鰭": 43004,
-      "葢": 43005,
-      "饒": 43006,
-      "礦": 43007,
-      "黽": 43008,
-      "禽": 43009,
-      "殉": 43010,
-      "竊": 43011,
-      "酋": 43012,
-      "呵": 43013,
-      "肅": 43014,
-      "勸": 43015,
-      "窘": 43016,
-      "諂": 43017,
-      "瀑": 43018,
-      "舒": 43019,
-      "籌": 43020,
-      "僨": 43021,
-      "恣": 43022,
-      "耆": 43023,
-      "悖": 43024,
-      "豈": 43025,
-      "尤": 43026,
-      "靱": 43027,
-      "瑟": 43028,
-      "媢": 43029,
-      "汀": 43030,
-      "窩": 43031,
-      "畢": 43032,
-      "熾": 43033,
-      "灼": 43034,
-      "厘": 43035,
-      "轍": 43036,
-      "橙": 43037,
-      "隱": 43038,
-      "舅": 43039,
-      "衷": 43040,
-      "貽": 43041,
-      "宍": 43042,
-      "闡": 43043,
-      "嗚": 43044,
-      "壯": 43045,
-      "恆": 43046,
-      "殷": 43047,
-      "覊": 43048,
-      "麾": 43049,
-      "痍": 43050,
-      "欵": 43051,
-      "鋒": 43052,
-      "揖": 43053,
-      "恰": 43054,
-      "歟": 43055,
-      "啜": 43056,
-      "攘": 43057,
-      "儁": 43058,
-      "葦": 43059,
-      "畏": 43060,
-      "襄": 43061,
-      "咤": 43062,
-      "饗": 43063,
-      "禊": 43064,
-      "孟": 43065,
-      "聽": 43066,
-      "遑": 43067,
-      "營": 43068,
-      "徽": 43069,
-      "鄙": 43070,
-      "俄": 43071,
-      "歐": 43072,
-      "羞": 43073,
-      "黃": 43074,
-      "戲": 43075,
-      "壹": 43076,
-      "尹": 43077,
-      "摧": 43078,
-      "菰": 43079,
-      "舜": 43080,
-      "雞": 43081,
-      "闊": 43082,
-      "恙": 43083,
-      "賂": 43084,
-      "蘊": 43085,
-      "羹": 43086,
-      "龕": 43087,
-      "爨": 43088,
-      "扈": 43089,
-      "抔": 43090,
-      "茲": 43091,
-      "貶": 43092,
-      "擧": 43093,
-      "謨": 43094,
-      "寃": 43095,
-      "濶": 43096,
-      "弐": 43097,
-      "醗": 43098,
-      "匍": 43099,
-      "斤": 43100,
-      "殲": 43101,
-      "頗": 43102,
-      "闢": 43103,
-      "僥": 43104,
-      "詔": 43105,
-      "鐵": 43106,
-      "懥": 43107,
-      "戮": 43108,
-      "趨": 43109,
-      "僩": 43110,
-      "儘": 43111,
-      "餘": 43112,
-      "肚": 43113,
-      "跋": 43114,
-      "續": 43115,
-      "勒": 43116,
-      "儡": 43117,
-      "揆": 43118,
-      "藥": 43119,
-      "糀": 43120,
-      "寵": 43121,
-      "嚴": 43122,
-      "矜": 43123,
-      "翰": 43124,
-      "唖": 43125,
-      "菉": 43126,
-      "蹈": 43127,
-      "倹": 43128,
-      "伍": 43129,
-      "傀": 43130,
-      "鵠": 43131,
-      "陷": 43132,
-      "據": 43133,
-      "竝": 43134,
-      "畧": 43135,
-      "戊": 43136,
-      "庸": 43137,
-      "樣": 43138,
-      "璽": 43139,
-      "拮": 43140,
-      "觸": 43141,
-      "隨": 43142,
-      "姦": 43143,
-      "宥": 43144,
-      "僇": 43145,
-      "賈": 43146,
-      "淺": 43147,
-      "柘": 43148,
-      "哨": 43149,
-      "擾": 43150,
-      "隧": 43151,
-      "讓": 43152,
-      "臀": 43153,
-      "慘": 43154,
-      "竭": 43155,
-      "聯": 43156,
-      "耘": 43157,
-      "罷": 43158,
-      "壓": 43159,
-      "愼": 43160,
-      "敖": 43161,
-      "朕": 43162,
-      "傚": 43163,
-      "蓁": 43164,
-      "蔗": 43165,
-      "駑": 43166,
-      "咀": 43167,
-      "榮": 43168,
-      "奧": 43169,
-      "裨": 43170,
-      "貮": 43171,
-      "埴": 43172,
-      "筭": 43173,
-      "黠": 43174,
-      "搖": 43175,
-      "倖": 43176,
-      "蘐": 43177,
-      "罕": 43178,
-      "袈": 43179,
-      "巓": 43180,
-      "盜": 43181,
-      "逕": 43182,
-      "聲": 43183,
-      "欷": 43184,
-      "譯": 43185,
-      "蓉": 43186,
-      "篩": 43187,
-      "辭": 43188,
-      "緡": 43189,
-      "纉": 43190,
-      "艱": 43191,
-      "熙": 43192,
-      "楷": 43193,
-      "雉": 43194,
-      "塡": 43195,
-      "岨": 43196,
-      "蕃": 43197,
-      "繼": 43198,
-      "價": 43199,
-      "諛": 43200,
-      "酉": 43201,
-      "雖": 43202,
-      "濤": 43203,
-      "簪": 43204,
-      "剝": 43205,
-      "逈": 43206,
-      "漸": 43207,
-      "縱": 43208,
-      "驅": 43209,
-      "觀": 43210,
-      "碇": 43211,
-      "盍": 43212,
-      "敎": 43213,
-      "縣": 43214,
-      "渺": 43215,
-      "羸": 43216,
-      "惟": 43217,
-      "逬": 43218,
-      "麿": 43219,
-      "棍": 43220,
-      "苅": 43221,
-      "恪": 43222,
-      "綬": 43223,
-      "遲": 43224,
-      "欣": 43225,
-      "辟": 43226,
-      "裴": 43227,
-      "岌": 43228,
-      "倩": 43229,
-      "壤": 43230,
-      "脹": 43231,
-      "孛": 43232,
-      "蹄": 43233,
-      "恕": 43234,
-      "舍": 43235,
       "錮": 43236,
-      "詭": 43237,
-      "猗": 43238,
-      "掩": 43239,
-      "醉": 43240,
-      "蜻": 43241,
-      "憊": 43242,
-      "桀": 43243,
-      "膽": 43244,
-      "毆": 43245,
-      "沮": 43246,
-      "勞": 43247,
-      "滿": 43248,
-      "洽": 43249,
-      "裔": 43250,
-      "亂": 43251,
-      "悉": 43252,
-      "鎭": 43253,
-      "蠻": 43254,
-      "菑": 43255,
-      "憮": 43256,
-      "榜": 43257,
-      "臘": 43258,
-      "恂": 43259,
-      "亦": 43260,
-      "苟": 43261,
-      "啻": 43262,
-      "吻": 43263,
-      "頒": 43264,
-      "證": 43265,
-      "畫": 43266,
-      "楯": 43267,
-      "滊": 43268,
-      "醵": 43269,
-      "禮": 43270,
-      "儕": 43271,
-      "頽": 43272,
-      "陋": 43273,
-      "煥": 43274,
-      "胥": 43275,
-      "咎": 43276,
-      "堯": 43277,
-      "誥": 43278,
-      "玻": 43279,
-      "臥": 43280,
-      "穆": 43281,
-      "蛉": 43282,
-      "囮": 43283,
-      "肆": 43284,
-      "妾": 43285,
-      "爰": 43286,
-      "屬": 43287,
-      "爭": 43288,
-      "贖": 43289,
-      "蹊": 43290,
-      "撹": 43291,
-      "忒": 43292,
-      "陪": 43293,
-      "誣": 43294,
-      "堡": 43295,
-      "寔": 43296,
-      "亨": 43297,
-      "綮": 43298,
-      "歸": 43299,
-      "豕": 43300,
-      "翹": 43301,
-      "鬢": 43302,
-      "騁": 43303,
-      "槙": 43304,
-      "隋": 43305,
-      "墾": 43306,
-      "緝": 43307,
-      "廢": 43308,
-      "駢": 43309,
-      "駭": 43310,
-      "烟": 43311,
-      "亥": 43312,
-      "竟": 43313,
-      "汝": 43314,
-      "疆": 43315,
-      "儼": 43316
     },
     "merges": [
       "▁ t",

     "end_of_word_suffix": null,
     "fuse_unk": true,
     "byte_fallback": false,
+    "ignore_merges": false,
     "vocab": {
       "<unk>": 0,
       "<s>": 1,
       "勅": 42797,
       "婿": 42798,
       "魯": 42799,
+      "禊": 42800,
+      "羞": 42801,
+      "瀑": 42802,
+      "斂": 42803,
+      "謁": 42804,
+      "諠": 42805,
+      "𠮟": 42806,
+      "蓉": 42807,
+      "悉": 42808,
+      "劾": 42809,
+      "逬": 42810,
+      "耘": 42811,
+      "裨": 42812,
+      "獻": 42813,
+      "逓": 42814,
+      "儕": 42815,
+      "聘": 42816,
+      "誥": 42817,
+      "悌": 42818,
+      "憮": 42819,
+      "盡": 42820,
+      "脩": 42821,
+      "詭": 42822,
+      "羣": 42823,
+      "賣": 42824,
+      "竢": 42825,
+      "痘": 42826,
+      "楯": 42827,
+      "畏": 42828,
+      "戮": 42829,
+      "飜": 42830,
+      "痍": 42831,
+      "恰": 42832,
+      "忒": 42833,
+      "縱": 42834,
+      "爭": 42835,
+      "悖": 42836,
+      "哨": 42837,
+      "苟": 42838,
+      "璽": 42839,
+      "攘": 42840,
+      "狡": 42841,
+      "恣": 42842,
+      "猗": 42843,
+      "黽": 42844,
+      "證": 42845,
+      "愈": 42846,
+      "賤": 42847,
+      "佩": 42848,
+      "畢": 42849,
+      "棍": 42850,
+      "筭": 42851,
+      "讓": 42852,
+      "頗": 42853,
+      "譬": 42854,
+      "馭": 42855,
+      "遁": 42856,
+      "專": 42857,
+      "紂": 42858,
+      "險": 42859,
+      "瑟": 42860,
+      "盈": 42861,
+      "傅": 42862,
+      "亥": 42863,
+      "墮": 42864,
+      "纉": 42865,
+      "伍": 42866,
+      "緡": 42867,
+      "倩": 42868,
+      "虧": 42869,
+      "赫": 42870,
+      "鎭": 42871,
+      "爲": 42872,
+      "饗": 42873,
+      "鳶": 42874,
+      "駢": 42875,
+      "駑": 42876,
+      "掩": 42877,
+      "洽": 42878,
+      "釀": 42879,
+      "逈": 42880,
+      "蹊": 42881,
+      "蜀": 42882,
+      "醨": 42883,
+      "爰": 42884,
+      "鄙": 42885,
+      "輕": 42886,
+      "楔": 42887,
+      "陋": 42888,
+      "騁": 42889,
+      "醗": 42890,
+      "猾": 42891,
+      "毘": 42892,
+      "咀": 42893,
+      "絜": 42894,
+      "淇": 42895,
+      "巌": 42896,
+      "榜": 42897,
+      "蛾": 42898,
+      "羸": 42899,
+      "敖": 42900,
+      "疆": 42901,
+      "驅": 42902,
+      "騷": 42903,
+      "趨": 42904,
+      "靜": 42905,
+      "營": 42906,
+      "詔": 42907,
+      "衷": 42908,
+      "廠": 42909,
+      "僨": 42910,
+      "殷": 42911,
+      "皈": 42912,
+      "厭": 42913,
+      "覺": 42914,
+      "聯": 42915,
+      "惟": 42916,
+      "竟": 42917,
+      "奧": 42918,
+      "辭": 42919,
+      "斥": 42920,
+      "撹": 42921,
+      "擾": 42922,
+      "浹": 42923,
+      "恂": 42924,
+      "儡": 42925,
+      "寔": 42926,
+      "瞻": 42927,
+      "舷": 42928,
+      "靱": 42929,
+      "鋒": 42930,
+      "雜": 42931,
+      "禽": 42932,
+      "拔": 42933,
+      "麾": 42934,
+      "檣": 42935,
+      "俄": 42936,
+      "彭": 42937,
+      "堡": 42938,
+      "壯": 42939,
+      "搆": 42940,
+      "恆": 42941,
+      "臥": 42942,
+      "諺": 42943,
+      "酉": 42944,
+      "扈": 42945,
+      "胤": 42946,
+      "苅": 42947,
+      "屡": 42948,
+      "膠": 42949,
+      "咤": 42950,
+      "嚴": 42951,
+      "彝": 42952,
+      "巓": 42953,
+      "濶": 42954,
+      "糢": 42955,
+      "攝": 42956,
+      "歸": 42957,
+      "賈": 42958,
+      "黠": 42959,
+      "貽": 42960,
+      "諮": 42961,
+      "譯": 42962,
+      "飮": 42963,
+      "胥": 42964,
+      "矜": 42965,
+      "儁": 42966,
+      "饜": 42967,
+      "筵": 42968,
+      "塡": 42969,
+      "蠻": 42970,
+      "蘐": 42971,
+      "臘": 42972,
+      "瑣": 42973,
+      "庸": 42974,
+      "擕": 42975,
+      "黨": 42976,
+      "僩": 42977,
+      "沮": 42978,
+      "窩": 42979,
+      "諫": 42980,
+      "蜻": 42981,
+      "摧": 42982,
+      "蛯": 42983,
+      "憾": 42984,
+      "厘": 42985,
+      "攬": 42986,
+      "搖": 42987,
+      "亂": 42988,
+      "峻": 42989,
+      "菑": 42990,
+      "憊": 42991,
+      "寃": 42992,
+      "珊": 42993,
+      "丞": 42994,
+      "毆": 42995,
+      "歡": 42996,
+      "籌": 42997,
+      "槓": 42998,
+      "淺": 42999,
+      "濟": 43000,
+      "兒": 43001,
+      "灼": 43002,
+      "舍": 43003,
+      "徽": 43004,
+      "豈": 43005,
+      "僇": 43006,
+      "蹄": 43007,
+      "壓": 43008,
+      "朧": 43009,
+      "儒": 43010,
+      "楷": 43011,
+      "禀": 43012,
+      "毋": 43013,
+      "廟": 43014,
+      "煥": 43015,
+      "岌": 43016,
+      "觸": 43017,
+      "簪": 43018,
+      "匐": 43019,
+      "薛": 43020,
+      "辜": 43021,
+      "巍": 43022,
+      "麿": 43023,
+      "皺": 43024,
+      "臀": 43025,
+      "頒": 43026,
+      "勞": 43027,
+      "虔": 43028,
+      "嚼": 43029,
+      "邑": 43030,
+      "嬰": 43031,
+      "勒": 43032,
+      "鬢": 43033,
+      "囮": 43034,
+      "誦": 43035,
+      "蓁": 43036,
+      "兩": 43037,
+      "篆": 43038,
+      "勸": 43039,
+      "隨": 43040,
+      "啜": 43041,
+      "舜": 43042,
+      "尸": 43043,
+      "茲": 43044,
+      "竭": 43045,
+      "岨": 43046,
+      "戲": 43047,
+      "陷": 43048,
+      "綮": 43049,
+      "歐": 43050,
+      "襄": 43051,
+      "廢": 43052,
+      "綬": 43053,
+      "戾": 43054,
+      "夭": 43055,
+      "遑": 43056,
+      "僞": 43057,
+      "駸": 43058,
+      "抔": 43059,
+      "耻": 43060,
+      "禎": 43061,
+      "畧": 43062,
+      "逼": 43063,
+      "圓": 43064,
+      "姦": 43065,
+      "憬": 43066,
+      "鑒": 43067,
+      "僻": 43068,
+      "鐸": 43069,
+      "濤": 43070,
+      "隋": 43071,
+      "鍾": 43072,
+      "滌": 43073,
+      "諛": 43074,
+      "挈": 43075,
+      "崗": 43076,
+      "匍": 43077,
+      "緝": 43078,
+      "澁": 43079,
+      "儼": 43080,
+      "僥": 43081,
+      "闢": 43082,
+      "聲": 43083,
+      "蛉": 43084,
+      "酋": 43085,
+      "恙": 43086,
+      "蹈": 43087,
+      "欵": 43088,
+      "唖": 43089,
+      "槙": 43090,
+      "裴": 43091,
+      "吻": 43092,
+      "蕃": 43093,
+      "熙": 43094,
+      "糀": 43095,
+      "彗": 43096,
+      "臾": 43097,
+      "寵": 43098,
+      "勵": 43099,
+      "懺": 43100,
+      "屬": 43101,
+      "鸞": 43102,
+      "隱": 43103,
+      "諂": 43104,
+      "尹": 43105,
+      "舒": 43106,
+      "餘": 43107,
+      "贖": 43108,
+      "嗚": 43109,
+      "闍": 43110,
+      "醉": 43111,
+      "縣": 43112,
+      "朕": 43113,
+      "彥": 43114,
+      "醵": 43115,
+      "殉": 43116,
+      "淫": 43117,
+      "恕": 43118,
+      "纂": 43119,
+      "錐": 43120,
+      "慄": 43121,
+      "玻": 43122,
+      "敎": 43123,
+      "袈": 43124,
+      "拮": 43125,
+      "裔": 43126,
+      "沃": 43127,
+      "雉": 43128,
+      "擔": 43129,
+      "闡": 43130,
+      "肚": 43131,
+      "陪": 43132,
+      "亦": 43133,
+      "渺": 43134,
+      "藥": 43135,
+      "鵠": 43136,
+      "忿": 43137,
+      "傚": 43138,
+      "謭": 43139,
+      "雖": 43140,
+      "胖": 43141,
+      "匈": 43142,
+      "黃": 43143,
+      "翰": 43144,
+      "埸": 43145,
+      "芒": 43146,
+      "葢": 43147,
+      "罕": 43148,
+      "肆": 43149,
+      "饒": 43150,
+      "諷": 43151,
+      "弐": 43152,
+      "讀": 43153,
+      "鵬": 43154,
+      "闊": 43155,
+      "盍": 43156,
+      "歟": 43157,
+      "俟": 43158,
+      "懥": 43159,
+      "埠": 43160,
+      "斤": 43161,
+      "龕": 43162,
+      "恤": 43163,
+      "舊": 43164,
+      "倹": 43165,
+      "諱": 43166,
+      "雞": 43167,
+      "孛": 43168,
+      "尤": 43169,
+      "團": 43170,
+      "汀": 43171,
+      "盜": 43172,
+      "篩": 43173,
+      "鐵": 43174,
+      "皓": 43175,
+      "揠": 43176,
+      "觀": 43177,
+      "羹": 43178,
+      "豕": 43179,
+      "帥": 43180,
+      "隧": 43181,
+      "獸": 43182,
+      "碇": 43183,
+      "殲": 43184,
+      "斷": 43185,
+      "桀": 43186,
+      "尙": 43187,
+      "愼": 43188,
+      "蔗": 43189,
+      "熾": 43190,
+      "續": 43191,
+      "皷": 43192,
+      "畫": 43193,
+      "丙": 43194,
+      "茫": 43195,
+      "揜": 43196,
+      "靡": 43197,
+      "墾": 43198,
+      "燮": 43199,
+      "逕": 43200,
+      "媢": 43201,
+      "穆": 43202,
+      "覊": 43203,
+      "漸": 43204,
+      "樣": 43205,
+      "恪": 43206,
+      "頴": 43207,
+      "咎": 43208,
+      "宥": 43209,
+      "礦": 43210,
+      "貶": 43211,
+      "膽": 43212,
+      "慘": 43213,
+      "貮": 43214,
+      "傀": 43215,
+      "懼": 43216,
+      "籾": 43217,
+      "吏": 43218,
+      "榮": 43219,
+      "繭": 43220,
+      "撿": 43221,
+      "甞": 43222,
+      "遲": 43223,
+      "驗": 43224,
+      "跋": 43225,
+      "滊": 43226,
+      "糾": 43227,
+      "諧": 43228,
+      "剝": 43229,
+      "肅": 43230,
+      "柘": 43231,
+      "辨": 43232,
+      "據": 43233,
+      "誣": 43234,
+      "竄": 43235,
       "錮": 43236,
+      "歎": 43237,
+      "窘": 43238,
+      "愧": 43239,
+      "聽": 43240,
+      "臂": 43241,
+      "妾": 43242,
+      "駭": 43243,
+      "頽": 43244,
+      "鏃": 43245,
+      "亨": 43246,
+      "殘": 43247,
+      "壹": 43248,
+      "轍": 43249,
+      "嘲": 43250,
+      "爨": 43251,
+      "驕": 43252,
+      "滿": 43253,
+      "碩": 43254,
+      "碍": 43255,
+      "虞": 43256,
+      "賂": 43257,
+      "脹": 43258,
+      "頰": 43259,
+      "恊": 43260,
+      "揆": 43261,
+      "躓": 43262,
+      "辟": 43263,
+      "倖": 43264,
+      "謨": 43265,
+      "梃": 43266,
+      "罷": 43267,
+      "覽": 43268,
+      "宍": 43269,
+      "諟": 43270,
+      "欷": 43271,
+      "繼": 43272,
+      "歔": 43273,
+      "埴": 43274,
+      "舅": 43275,
+      "啻": 43276,
+      "戊": 43277,
+      "儘": 43278,
+      "處": 43279,
+      "巖": 43280,
+      "豫": 43281,
+      "擧": 43282,
+      "竊": 43283,
+      "狸": 43284,
+      "蘊": 43285,
+      "橙": 43286,
+      "鰭": 43287,
+      "甍": 43288,
+      "菰": 43289,
+      "價": 43290,
+      "揖": 43291,
+      "嫡": 43292,
+      "耆": 43293,
+      "葦": 43294,
+      "堯": 43295,
+      "戌": 43296,
+      "獨": 43297,
+      "汝": 43298,
+      "澳": 43299,
+      "呵": 43300,
+      "憚": 43301,
+      "嚮": 43302,
+      "翹": 43303,
+      "孟": 43304,
+      "鄭": 43305,
+      "壤": 43306,
+      "竝": 43307,
+      "菉": 43308,
+      "烟": 43309,
+      "閻": 43310,
+      "禮": 43311,
+      "犂": 43312,
+      "欣": 43313,
+      "惡": 43314,
+      "鉤": 43315,
+      "艱": 43316
     },
     "merges": [
       "▁ t",

tokenizer_config.json CHANGED Viewed

@@ -1,6 +1,7 @@
 {
   "add_bos_token": true,
   "add_eos_token": false,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
@@ -31,6 +32,7 @@
   "clean_up_tokenization_spaces": false,
   "cls_token": "<s>",
   "eos_token": "</s>",
   "mask_token": "<unk>",
   "model_max_length": 4096,
   "pad_token": "</s>",

 {
   "add_bos_token": true,
   "add_eos_token": false,
+  "add_prefix_space": null,
   "added_tokens_decoder": {
     "0": {
       "content": "<unk>",
   "clean_up_tokenization_spaces": false,
   "cls_token": "<s>",
   "eos_token": "</s>",
+  "legacy": true,
   "mask_token": "<unk>",
   "model_max_length": 4096,
   "pad_token": "</s>",

upos.py CHANGED Viewed

@@ -1,5 +1,4 @@
-from transformers import TokenClassificationPipeline,MistralModel,MistralPreTrainedModel
-from transformers.modeling_outputs import TokenClassifierOutput
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
@@ -40,41 +39,3 @@ class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w
-class RawTokenClassificationPipeline(TokenClassificationPipeline):
-  def check_model_type(self,supported_models):
-    pass
-class MistralForTokenClassification(MistralPreTrainedModel):
-  def __init__(self,config):
-    from torch import nn
-    super().__init__(config)
-    self.num_labels=config.num_labels
-    self.model=MistralModel(config)
-    if hasattr(config,"classifier_dropout") and config.classifier_dropout is not None:
-      classifier_dropout=config.classifier_dropout
-    elif hasattr(config,"hidden_dropout") and config.hidden_dropout is not None:
-      classifier_dropout=config.hidden_dropout
-    else:
-      classifier_dropout=0.1
-    self.dropout=nn.Dropout(classifier_dropout)
-    self.classifier=nn.Linear(config.hidden_size,config.num_labels)
-    self.post_init()
-  def get_input_embeddings(self):
-    return self.model.embed_tokens
-  def set_input_embeddings(self,value):
-    self.model.embed_tokens=value
-  def forward(self,input_ids=None,past_key_values=None,attention_mask=None,position_ids=None,inputs_embeds=None,labels=None,use_cache=None,output_attentions=None,output_hidden_states=None,return_dict=None):
-    return_dict=return_dict if return_dict is not None else self.config.use_return_dict
-    transformer_outputs=self.model(input_ids,past_key_values=past_key_values,attention_mask=attention_mask,position_ids=position_ids,inputs_embeds=inputs_embeds,use_cache=use_cache,output_attentions=output_attentions,output_hidden_states=output_hidden_states,return_dict=return_dict)
-    hidden_states=transformer_outputs[0]
-    hidden_states=self.dropout(hidden_states)
-    logits=self.classifier(hidden_states)
-    loss=None
-    if labels is not None:
-      from torch import nn
-      loss_fct=nn.CrossEntropyLoss()
-      loss=loss_fct(logits.view(-1,self.num_labels),labels.view(-1))
-    if not return_dict:
-      output=(logits,)+transformer_outputs[2:]
-      return ((loss,)+output) if loss is not None else output
-    return TokenClassifierOutput(loss=loss,logits=logits,hidden_states=transformer_outputs.hidden_states,attentions=transformer_outputs.attentions)

+from transformers import TokenClassificationPipeline
 class BellmanFordTokenClassificationPipeline(TokenClassificationPipeline):
   def __init__(self,**kwargs):
       t["text"]=model_outputs["sentence"][t["start"]:t["end"]]
     return w