Spaces:
Build error
Build error
File size: 3,515 Bytes
83d5adb e149b0f 83d5adb 8f1745b c106121 e149b0f 83d5adb f16a715 83d5adb c106121 e149b0f c106121 e149b0f 83d5adb 58e8b0b 83d5adb e149b0f 83d5adb e149b0f 83d5adb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 |
import os
import re
import argparse
import pickle
import tokenizers
import qarac.corpora.BNCorpus
import qarac.corpora.Batcher
import qarac.models.qarac_base_model
import keras
import tensorflow
import spacy
import pandas
import qarac.utils.CoreferenceResolver
def decoder_loss(y_true,y_pred):
return keras.losses.sparse_categorical_crossentropy(y_true,
y_pred.logits,
logits=True)
def capitalise(token,i):
return token.text_with_ws.title() if i==0 or token.tag_.startswith('NNP') else token.text_with_ws.lower()
def clean_question(doc):
words = [capitalise(token,i) for (i,token) in enumerate(doc)]
if words[-1]!='?':
words.append('?')
return ''.join(words)
def prepare_wiki_qa(filename,outfilename):
data = pandas.read_csv(filename,sep='\t')
data['QNum']=data['QuestionID'].apply(lambda x: int(x[1:]))
nlp = spacy.load('en_core_web_trf')
predictor = qarac.utils.CoreferenceResolver.CoreferenceResolver()
data['Resolved_answer'] = data.groupby('QNum')['Sentence'].transform(predictor)
unique_questions = data.groupby('QNum')['Question'].first()
cleaned_questions = pandas.Series([clean_question(doc)
for doc in nlp.pipe(unique_questions)],
index = unique_questions.index)
for (i,question) in cleaned_questions.items():
data.loc[data['QNum']==i,'Cleaned_question']=question
data[['Cleaned_question','Resolved_answer','Label']].to_csv(outfilename)
def train_base_model(task,filename):
tokenizer = tokenizers.Tokenizer.from_pretrained('xlm-roberta-base')
tokenizer.add_special_tokens(['<start>','<end>','<pad>'])
tokenizer.save('/'.join([os.environ['HOME'],
'QARAC',
'models',
'tokenizer.json']))
bnc = qarac.corpora.BNCorpus.BNCorpus(tokenizer=tokenizer,
task=task)
(train,test)=bnc.split(0.01)
train_data=qarac.corpora.Batcher.Batcher(train)
model = qarac.models.qarac_base_model.qarac_base_model(tokenizer.get_vocab_size(),
768,
12,
task=='decode')
optimizer = keras.optimizers.Nadam(learning_rate=keras.optimizers.schedules.ExponentialDecay(1.0e-5, 100, 0.99))
model.compile(optimizer=optimizer,loss='sparse_categorical_crossentropy',metrics='accuracy')
model.fit(train_data,
epochs=100,
workers = 16,
use_multiprocessing=True)
test_data=qarac.corpora.Batcher.Batcher(test)
print(model.evaluate(test_data))
model.save(filename)
if __name__ == '__main__':
parser = argparse.ArgumentParser(prog='QARAC',
description='Experimental NLP system, aimed at improving factual accuracy')
parser.add_argument('task')
parser.add_argument('-f','--filename')
parser.add_argument('-t','--training-task')
parser.add_argument('-o','--outputfile')
args = parser.parse_args()
if args.task == 'train_base_model':
train_base_model(args.training_task,args.filename)
elif args.task == 'prepare_wiki_qa':
prepare_wiki_qa(args.filename,args.outputfile)
|