Spaces:
Build error
Build error
| import re | |
| from copy import deepcopy | |
| import transformers | |
| from transformers import BartTokenizer | |
| import jsonlines | |
| import json | |
| # dict = {"rel_triggers": [], "gold_rel_links": [], "doc_key": "nw_RC00c8620ef5810429342a1c339e6c76c1b0b9add3f6010f04482fd832", "ent_spans": [[27, 27, [["evt043arg01communicator", 1.0]]], [48, 48, [["evt043arg03place", 1.0]]], [32, 36, [["evt043arg02recipient", 1.0]]]], "language_id": "eng", "source_url": "http://bbc.co.uk/sport/athletics/36295481", "evt_triggers": [[31, 31, [["contact.prevarication.broadcast", 1.0]]]], "split": "test", "sentences": [["We", "are", "ashamed", "of", "them", ".", "\""], ["However", ",", "Mutko", "stopped", "short", "of", "admitting", "the", "doping", "scandal", "was", "state", "sponsored", "."], ["\"", "We", "are", "very", "sorry", "that", "athletes", "who", "tried", "to", "deceive", "us", ",", "and", "the", "world", ",", "were", "not", "caught", "sooner", "."], ["We", "are", "very", "sorry", "because", "Russia", "is", "committed", "to", "upholding", "the", "highest", "standards", "in", "sport", "and", "is", "opposed", "to", "anything", "that", "threatens", "the", "Olympic", "values", ",", "\"", "he", "said", "."], ["English", "former", "heptathlete", "and", "Athens", "2004", "bronze", "medallist", "Kelly", "Sotherton", "was", "unhappy", "with", "Mutko", "'s", "plea", "for", "Russia", "'s", "ban", "to", "be", "lifted", "for", "Rio"]], "gold_evt_links": [[[31, 31], [27, 27], "evt043arg01communicator"], [[31, 31], [32, 32], "evt043arg02recipient"], [[31, 31], [48, 48], "evt043arg03place"]], "clusters": [[[0, 0], [22, 22], [32, 32], [43, 43], [48, 48], [90, 91]], [[9, 9], [70, 70], [86, 87]]], "corefs": [[[0, 0], [22, 22], [32, 32], [43, 43], [48, 48], [90, 90]], [[9, 9], [70, 70], [86, 86]]]} | |
| # | |
| # template = "what is the <arg>" | |
| # | |
| # context_words = [w for sent in dict['sentences'] for w in sent] | |
| # | |
| # argtext = context_words[27] | |
| # | |
| # print(argtext) | |
| # | |
| # template = re.sub('<arg>', argtext, template) | |
| # | |
| # print(template) | |
| # for lidx, triple in enumerate(dict['gold_evt_links']): | |
| # # 触发词 论元 论元 | |
| # # 例子： "gold_evt_links": | |
| # # [[[40, 40], [33, 33], "evt089arg01victim"], | |
| # # [[40, 40], [28, 28], "evt089arg02place"]] | |
| # trigger_span, argument_span, arg_name = triple | |
| # # 第几个论元 | |
| # arg_num = ontology_dict[evt_type.replace('n/a', 'unspecified')][arg_name] | |
| # # 具体论元内容 短语 | |
| # arg_text = ' '.join(context_words[argument_span[0]:argument_span[1] + 1]) | |
| # # 通过正则表达式的方式将模板中的每个<arg> 替换为具体的论元内容 | |
| # # 按照顺序将列表中的<arg>依次替换为 | |
| # template[lidx] = re.sub('<{}>'.format(arg_num), arg_text, template[lidx]) | |
| # # "ent_spans": [[27, 27, [["evt043arg01communicator", 1.0]]], | |
| # # [48, 48, [["evt043arg03place", 1.0]]], | |
| # # [32, 36, [["evt043arg02recipient", 1.0]]]] | |
| # context_words = [w for sent in dict['sentences'] for w in sent] | |
| # | |
| # print(context_words[32]) | |
| # print(context_words[33]) | |
| # print(context_words[34]) | |
| # print(context_words[35]) | |
| # print(context_words[36]) | |
| def get_event_type(ex): | |
| evt_type = [] | |
| for evt in ex['evt_triggers']: | |
| for t in evt[2]: | |
| evt_type.append(t[0]) | |
| return evt_type | |
| def create_gold_gen(ex, ontology_dict, mark_trigger=True): | |
| # 设置三个总列表、存放输入模板、输出模板 | |
| # 设置三个总列表、存放输入模板、输出模板 | |
| INPUT = [] | |
| OUTPUT = [] | |
| CONTEXT = [] | |
| evt_type = get_event_type(ex)[0] | |
| context_words = [w for sent in ex['sentences'] for w in sent] | |
| # print(context_words[48]) | |
| input_template = ontology_dict[evt_type.replace('n/a', 'unspecified')]['template'] | |
| i = len(input_template) | |
| input_list = [] | |
| for x in range(i): | |
| str = re.sub(r'<arg\d>', '<arg>', input_template[x]) | |
| input_list.append(str) | |
| # 其中input_list种存放的是 原始数据中<arg1> 全部替换为 <arg> 之后的模板 下一步应该进行分词 | |
| # temp = [] | |
| for x in range(i): | |
| space_tokenized_template = input_list[x].split(' ') | |
| INPUT.append(space_tokenized_template) | |
| space_tokenized_template = [] | |
| # 其中temp中存放的都是分词后的模板 下一步对temp中的所有元素进行tokenize | |
| tokenized_input_template = [] | |
| tokenizer = BartTokenizer.from_pretrained('facebook/bart-large') | |
| temp = [] | |
| for x in range(len(INPUT)): | |
| for w in INPUT[x]: | |
| tokenized_input_template.extend(tokenizer.tokenize(w, add_prefix_space=True)) | |
| # print(tokenized_input_template) | |
| temp.append(tokenized_input_template) | |
| tokenized_input_template = [] | |
| print(temp) | |
| break | |
| template = ontology_dict[evt_type.replace('n/a', 'unspecified')]['template'] | |
| # if ex['doc_key'] == 'nw_RC04992035300b2ec94d8692646a28dc8b5f210c94842d20834c5342df': | |
| # print('bbb') | |
| # ['<arg1> was injured by injurer with medical issue at place', | |
| # 'Victim was injured by <arg2> with medical issue at place', | |
| # 'Victim was injured by injurer with medical issue at <arg3> place', | |
| # 'Victim was injured by injurer with <arg4> medical issue at place'] | |
| # print(template) | |
| for lidx, triple in enumerate(ex['gold_evt_links']): | |
| # 触发词 论元 论元 | |
| # 例子： "gold_evt_links": | |
| # [[50, 50], [48, 48], 'evt092arg01victim'] | |
| # [[50, 50], [7, 7], 'evt092arg03place'] | |
| trigger_span, argument_span, arg_name = triple | |
| if ex['doc_key'] == 'nw_RC013c8e78b7e8a4fb22193483877058f712dfd8b75b7a06d950de0b8f': | |
| print(len(ex['gold_evt_links'])) | |
| # 第几个论元 | |
| arg_num = ontology_dict[evt_type.replace('n/a', 'unspecified')][arg_name] | |
| # 具体论元内容 短语 | |
| arg_text = ' '.join(context_words[argument_span[0]:argument_span[1] + 1]) | |
| # if ex['doc_key'] == 'nw_RC04992035300b2ec94d8692646a28dc8b5f210c94842d20834c5342df': | |
| # print(arg_num) | |
| # print(arg_text) | |
| # print(arg_text) | |
| # 通过正则表达式的方式将模板中的每个<arg> 替换为具体的论元内容 | |
| # 搜索templat中的arg_num 找到对应的序列 | |
| # INDEX = 0 | |
| for index in range(len(template)): | |
| if arg_num in template[index]: | |
| break | |
| else: | |
| continue | |
| # INDEX += 1 | |
| template[index] = re.sub('<{}>'.format(arg_num), arg_text, template[index]) | |
| if ex['doc_key'] == 'nw_RC013c8e78b7e8a4fb22193483877058f712dfd8b75b7a06d950de0b8f': | |
| print('aaa') | |
| print(template) | |
| trigger = ex['evt_triggers'][0] | |
| # 将context放入CONTEXT中 | |
| for w in range(i): | |
| CONTEXT.append(context_words) | |
| output_template = [] | |
| # 此时的template中已经全部替换为论元短语 这部是将<arg1> 替换为<arg> | |
| for i in range(len(template)): | |
| output_template.append(re.sub(r'<arg\d>', '<arg>', template[i])) | |
| # spaceout_tokenized_template = [] | |
| for i in range(len(output_template)): | |
| OUTPUT.append(output_template[i].split(' ')) | |
| # tokenized_out_template = [] | |
| # for i in range(len(spaceout_tokenized_template)): | |
| # for w in spaceout_tokenized_template[i]: | |
| # tokenized_out_template.extend(self.tokenizer.tokenize(w, add_prefix_space=True)) | |
| # OUTPUT.append(tokenized_out_template) | |
| # tokenized_out_template = [] | |
| return INPUT, OUTPUT, CONTEXT | |
| def load_ontology(): | |
| ontology_dict = {} | |
| with open('aida_ontology_fj-5.csv', 'r') as f: | |
| for lidx, line in enumerate(f): | |
| if lidx == 0: # header | |
| continue | |
| fields = line.strip().split(',') | |
| if len(fields) < 2: | |
| break | |
| evt_type = fields[0] | |
| if evt_type in ontology_dict.keys(): | |
| # 得到该事件类型下的所有论元类型 | |
| args = fields[2:] | |
| # 将该事件类型对应的模板中的论元模板 填充到onto_logy字典中 | |
| ontology_dict[evt_type]['template'].append(fields[1]) | |
| for i, arg in enumerate(args): | |
| if arg != '': | |
| # 事件类型下添加字典一项 arg1的值为arg | |
| ontology_dict[evt_type]['arg{}'.format(i + 1)] = arg | |
| ontology_dict[evt_type][arg] = 'arg{}'.format(i + 1) | |
| # 即扫描到的事件类型在 evt_type_dict.keys() 还未存在过 | |
| else: | |
| # 建立该事件类型的key | |
| ontology_dict[evt_type] = {} | |
| args = fields[2:] | |
| ontology_dict[evt_type]['template'] = [] | |
| ontology_dict[evt_type]['template'].append(fields[1]) | |
| for i, arg in enumerate(args): | |
| if arg != '': | |
| # 事件类型下添加字典一项 arg1的值为arg | |
| ontology_dict[evt_type]['arg{}'.format(i + 1)] = arg | |
| ontology_dict[evt_type][arg] = 'arg{}'.format(i + 1) | |
| return ontology_dict | |
| def prepare_data(): | |
| ontology_dict = load_ontology() | |
| # ('train', './data/RAMS_1.0/data/train.jsonlines'), | |
| # ('test', './data/RAMS_1.0/data/test_head_coref.jsonlines') | |
| for split, f in [('val', './data/RAMS_1.0/data/dev.jsonlines'),('train', './data/RAMS_1.0/data/train.jsonlines'), | |
| ('test', './data/RAMS_1.0/data/test_head_coref.jsonlines')]: | |
| # , open('head_templates_preprocessed_data/{}.jsonl'.format(split), 'w') as writer | |
| with open(f, 'r') as reader: | |
| # print(ontology_dict['contact.prevarication.broadcast']) | |
| for lidx, line in enumerate(reader): | |
| ex = json.loads(line.strip()) | |
| # print(lidx) | |
| # print(ex) | |
| event_type = get_event_type(ex)[0] | |
| if ex['doc_key'] == 'nw_RC013c8e78b7e8a4fb22193483877058f712dfd8b75b7a06d950de0b8f': | |
| # {'rel_triggers': [], 'gold_rel_links': [], | |
| # 'doc_key': 'nw_RC04992035300b2ec94d8692646a28dc8b5f210c94842d20834c5342df', | |
| # 'ent_spans': [[48, 48, [['evt092arg01victim', 1.0]]], [7, 7, [['evt092arg03place', 1.0]]]], | |
| # 'language_id': 'eng', | |
| # 'source_url': 'http://news.sky.com/story/attack-in-nice-truck-ploughes-into-crowd-10502068', | |
| # 'evt_triggers': [[50, 50, [['life.injure.n/a', 1.0]]]], 'split': 'test', | |
| # 'sentences': [[':', ':', 'History', 'Of', 'Deadly', 'Attacks', 'In', 'France'], | |
| # ['One', ',', 'Laurence', 'Olding', ',', 'was', 'with', 'his', 'fiance', 'and', | |
| # 'jumped', 'over', 'a', 'promenade', 'wall', 'onto', 'the', 'concrete', 'below', | |
| # 'to', 'avoid', 'the', 'truck', '.'], | |
| # ['Emerging', 'from', 'hospital', 'in', 'bandages', 'he', 'said', ':', '"', 'There', | |
| # 'was', 'debris', 'in', 'the', 'streets', ',', 'people', 'lying', 'injured', 'or', | |
| # 'dead', 'in', 'the', 'road', '.', '"'], | |
| # ['Video', ':', 'Hollande', 'On', "'", 'Horror', "'", 'Of', 'Attack'], | |
| # ['Two', 'Americans', '-', 'Sean', 'Copeland', 'and', 'his', '11-year', '-', 'old', | |
| # 'son', 'Brodie', 'from', 'Texas', '-', 'have', 'been', 'confirmed', 'among', 'the', | |
| # 'dead', ',', 'a', 'US', 'official', 'said', '.']], | |
| # 'gold_evt_links': [[[50, 50], [48, 48], 'evt092arg01victim'], | |
| # [[50, 50], [7, 7], 'evt092arg03place']], | |
| # 'clusters': [[[10, 11], [15, 15], [37, 37]], [[70, 71], [73, 73]]], | |
| # 'corefs': [[[11, 11], [15, 15], [37, 37]], [[71, 71], [73, 73]]]} | |
| print(ex) | |
| # {'template': ['<arg1> was injured by injurer with medical issue at place', | |
| # 'Victim was injured by <arg2> with medical issue at place', | |
| # 'Victim was injured by injurer with medical issue at <arg3> place', | |
| # 'Victim was injured by injurer with <arg4> medical issue at place'], | |
| # 'arg1': 'evt092arg01victim', 'evt092arg01victim': 'arg1', 'arg2': 'evt092arg02injurer', | |
| # 'evt092arg02injurer': 'arg2', 'arg3': 'evt092arg03place', 'evt092arg03place': 'arg3', | |
| # 'arg4': 'evt092arg04medicalissue', 'evt092arg04medicalissue': 'arg4'} | |
| print(ontology_dict[event_type.replace('n/a','unspecified')]) | |
| input_template, output_template, context = create_gold_gen(ex, ontology_dict, | |
| True) | |
| ontology_dict = load_ontology() | |
| if ex['doc_key'] == 'nw_RC013c8e78b7e8a4fb22193483877058f712dfd8b75b7a06d950de0b8f': | |
| # [['<arg>', 'was', 'injured', 'by', 'injurer', 'with', 'medical', 'issue', 'at', 'place'], | |
| # ['Victim', 'was', 'injured', 'by', '<arg>', 'with', 'medical', 'issue', 'at', 'place'], | |
| # ['Victim', 'was', 'injured', 'by', 'injurer', 'with', 'medical', 'issue', 'at', '<arg>', 'place'], | |
| # ['Victim', 'was', 'injured', 'by', 'injurer', 'with', '<arg>', 'medical', 'issue', 'at', 'place']] | |
| print(input_template) | |
| # [['people', 'was', 'injured', 'by', 'injurer', 'with', 'medical', 'issue', 'at', 'place'], | |
| # ['Victim', 'was', 'injured', 'by', '<arg>', 'with', 'medical', 'issue', 'at', 'place'], | |
| # ['Victim', 'was', 'injured', 'by', 'injurer', 'with', 'medical', 'issue', 'at', '<arg>', 'place'], | |
| # ['Victim', 'was', 'injured', 'by', 'injurer', 'with', '<arg>', 'medical', 'issue', 'at', 'place']] | |
| print(output_template) | |
| # print(input_template) | |
| # 4 4 4 | |
| # print(len(input_template)) | |
| # print(len(output_template)) | |
| # print(len(context)) | |
| [[':', ':', 'History', 'Of', 'Deadly', 'Attacks', 'In', 'France'], | |
| ['One', ',', 'Laurence', 'Olding', ',', 'was', 'with', 'his', 'fiance', 'and', 'jumped', 'over', 'a', 'promenade', 'wall', 'onto', 'the', 'concrete', 'below', 'to', 'avoid', 'the', 'truck', '.'], | |
| ['Emerging', 'from', 'hospital', 'in', 'bandages', 'he', 'said', ':', '"', 'There', 'was', 'debris', 'in', 'the', 'streets', ',', 'people', 'lying', 'injured', 'or', 'dead', 'in', 'the', 'road', '.', '"'], | |
| ['Video', ':', 'Hollande', 'On', "'", 'Horror', "'", 'Of', 'Attack'], | |
| ['Two', 'Americans', '-', 'Sean', 'Copeland', 'and', 'his', '11-year', '-', 'old', 'son', 'Brodie', 'from', 'Texas', '-', 'have', 'been', 'confirmed', 'among', 'the', 'dead', ',', 'a', 'US', 'official', 'said', '.']] | |
| [':', ':', 'History', 'Of', 'Deadly', 'Attacks', 'In', 'France', | |
| 'One', ',', 'Laurence', 'Olding', ',', 'was', 'with', 'his', 'fiance', 'and', 'jumped', 'over', 'a', 'promenade', 'wall', 'onto', 'the', 'concrete', 'below', 'to', 'avoid', 'the', 'truck', '.', | |
| 'Emerging', 'from', 'hospital', 'in', 'bandages', 'he', 'said', ':', '"', 'There', 'was', 'debris', 'in', 'the', 'streets', ',', 'people', 'lying', 'injured', 'or', 'dead', 'in', 'the', 'road', '.', '"', | |
| 'Video', ':', 'Hollande', 'On', "'", 'Horror', "'", 'Of', 'Attack', | |
| 'Two', 'Americans', '-', 'Sean', 'Copeland', 'and', 'his', '11-year', '-', 'old', 'son', 'Brodie', 'from', 'Texas', '-', 'have', 'been', 'confirmed', 'among', 'the', 'dead', ',', 'a', 'US', 'official', 'said', '.'] | |
| prepare_data() | |