model improved

maker.py

@@ -5,7 +5,7 @@ url="https://github.com/KoichiYasuoka/spaCy-Thai"
 import os
 d=os.path.join(os.path.basename(url),"UD_Thai-Corpora")
 os.system("test -d {} || git clone --depth=1 {}".format(d,url))
-s='{if(NF>0)u=u$0"\\n";else{if(u~/\\t0\\troot\\t/)print u>"train.conllu";u=""}}'
+s='{if(NF>0)u=u$0"\\n";else{f=FILENAME;if(u~/\\t0\\troot\\t/)print u>(f~/-dev/?"dev":f~/-test/?"test":"train")".conllu";u=""}}'
 os.system("nawk -F'\\t' '{}' {}/*-ud-*.conllu".format(s,d))
 class UDgoeswithDataset(object):
   def __init__(self,conllu,tokenizer):
@@ -33,15 +33,23 @@ class UDgoeswithDataset(object):
             self.tags.append([dep]+[t if h[j]==i+1 else dep for j,t in enumerate(p)]+[dep,dep])
           c=[]
     self.label2id={l:i for i,l in enumerate(sorted(label))}
+  def __call__(*args):
+    label=set(sum([list(t.label2id) for t in args],[]))
+    lid={l:i for i,l in enumerate(sorted(label))}
+    for t in args:
+      t.label2id=lid
+    return lid
   __len__=lambda self:len(self.ids)
   __getitem__=lambda self,i:{"input_ids":self.ids[i],"labels":[self.label2id[t] for t in self.tags[i]]}
 from transformers import AutoTokenizer,AutoConfig,AutoModelForTokenClassification,DataCollatorForTokenClassification,TrainingArguments,Trainer
 tkz=AutoTokenizer.from_pretrained(src)
 trainDS=UDgoeswithDataset("train.conllu",tkz)
-lid=trainDS.label2id
+devDS=UDgoeswithDataset("dev.conllu",tkz)
+testDS=UDgoeswithDataset("test.conllu",tkz)
+lid=trainDS(devDS,testDS)
 cfg=AutoConfig.from_pretrained(src,num_labels=len(lid),label2id=lid,id2label={i:l for l,i in lid.items()})
-arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,learning_rate=5e-05,warmup_ratio=0.1)
-trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg),train_dataset=trainDS)
+arg=TrainingArguments(num_train_epochs=3,per_device_train_batch_size=32,output_dir="/tmp",overwrite_output_dir=True,save_total_limit=2,evaluation_strategy="epoch",learning_rate=5e-05,warmup_ratio=0.1)
+trn=Trainer(args=arg,data_collator=DataCollatorForTokenClassification(tkz),model=AutoModelForTokenClassification.from_pretrained(src,config=cfg),train_dataset=trainDS,eval_dataset=devDS)
 trn.train()
 trn.save_model(tgt)
 tkz.save_pretrained(tgt)

pytorch_model.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:990574f354bc5c07915f67c322132900a90fa675991ebc32d1de2c39f03a34a8
+oid sha256:84a1018e4627ed153c908596557ebbd905a1dedeff2fb076e0ff62175bd55327
 size 351720561