import os import gradio as gr import subprocess import sys def install(package): subprocess.check_call([sys.executable, "-m", "pip", "install", package]) install("numpy") install("torch") install("transformers") install("unidecode") import numpy as np import torch from transformers import AutoTokenizer from transformers import BertForTokenClassification from collections import Counter from unidecode import unidecode import string import re auth_token = os.environ.get("AUTH_TOKEN") tokenizer = AutoTokenizer.from_pretrained("osiria/bert-base-cased-ner-en", token=auth_token) model = BertForTokenClassification.from_pretrained("osiria/bert-base-cased-ner-en", num_labels = 5, token=auth_token) device = torch.device("cpu") model = model.to(device) model.eval() from transformers import pipeline ner = pipeline('ner', model=model, tokenizer=tokenizer, device=-1) header = '''--------------------------------------------------------------------------------------------------
 D     E     M  O

''' maps = {"O": "NONE", "PER": "PER", "LOC": "LOC", "ORG": "ORG", "MISC": "MISC", "DATE": "DATE"} reg_month = "(?:gennaio|febbraio|marzo|aprile|maggio|giugno|luglio|agosto|settembre|ottobre|novembre|dicembre|january|february|march|april|may|june|july|august|september|october|november|december)" reg_date = "(?:\d{1,2}\°{0,1}|primo|\d{1,2}\º{0,1})" + " " + reg_month + " " + "\d{4}|" reg_date = reg_date + reg_month + " " + "\d{4}|" reg_date = reg_date + "\d{1,2}" + " " + reg_month reg_date = reg_date + "\d{1,2}" + "(?:\/|\.)\d{1,2}(?:\/|\.)" + "\d{4}|" reg_date = reg_date + "(?<=dal )\d{4}|(?<=al )\d{4}|(?<=nel )\d{4}|(?<=anno )\d{4}|(?<=del )\d{4}|" reg_date = reg_date + "\d{1,5} a\.c\.|\d{1,5} d\.c\." map_punct = {"’": "'", "«": '"', "»": '"', "”": '"', "“": '"', "–": "-", "$": ""} unk_tok = 9005 merge_th_1 = 0.8 merge_th_2 = 0.4 min_th = 0.5 def extract(text): text = text.strip() for mp in map_punct: text = text.replace(mp, map_punct[mp]) text = re.sub("\[\d+\]", "", text) warn_flag = False res_total = [] out_text = "" for p_text in text.split("\n"): if p_text: toks = tokenizer.encode(p_text) if unk_tok in toks: warn_flag = True res_orig = ner(p_text, aggregation_strategy = "first") res_orig = [el for r, el in enumerate(res_orig) if len(el["word"].strip()) > 1] res = [] for r, ent in enumerate(res_orig): if r > 0 and ent["score"] < merge_th_1 and ent["start"] <= res[-1]["end"] + 1 and ent["score"] <= res[-1]["score"]: res[-1]["word"] = res[-1]["word"] + " " + ent["word"] res[-1]["score"] = merge_th_1*(res[-1]["score"] > merge_th_2) res[-1]["end"] = ent["end"] elif r < len(res_orig) - 1 and ent["score"] < merge_th_1 and res_orig[r+1]["start"] <= ent["end"] + 1 and res_orig[r+1]["score"] > ent["score"]: res_orig[r+1]["word"] = ent["word"] + " " + res_orig[r+1]["word"] res_orig[r+1]["score"] = merge_th_1*(res_orig[r+1]["score"] > merge_th_2) res_orig[r+1]["start"] = ent["start"] else: res.append(ent) res = [el for r, el in enumerate(res) if el["score"] >= min_th] dates = [{"entity_group": "DATE", "score": 1.0, "word": p_text[el.span()[0]:el.span()[1]], "start": el.span()[0], "end": el.span()[1]} for el in re.finditer(reg_date, p_text, flags = re.IGNORECASE)] res.extend(dates) res = sorted(res, key = lambda t: t["start"]) res_total.extend(res) chunks = [("", "", 0, "NONE")] for el in res: if maps[el["entity_group"]] != "NONE": tag = maps[el["entity_group"]] chunks.append((p_text[el["start"]: el["end"]], p_text[chunks[-1][2]:el["end"]], el["end"], tag)) if chunks[-1][2] < len(p_text): chunks.append(("END", p_text[chunks[-1][2]:], -1, "NONE")) chunks = chunks[1:] n_text = [] for i, chunk in enumerate(chunks): rep = chunk[0] if chunk[3] == "PER": rep = 'ᴘᴇʀ ' + chunk[0] + '' elif chunk[3] == "LOC": rep = 'ʟᴏᴄ ' + chunk[0] + '' elif chunk[3] == "ORG": rep = 'ᴏʀɢ ' + chunk[0] + '' elif chunk[3] == "MISC": rep = 'ᴍɪsᴄ ' + chunk[0] + '' elif chunk[3] == "DATE": rep = 'ᴅᴀᴛᴇ ' + chunk[0] + '' n_text.append(chunk[1].replace(chunk[0], rep)) n_text = "".join(n_text) if out_text: out_text = out_text + "
" + n_text else: out_text = n_text tags = [el["word"] for el in res_total if el["entity_group"] not in ['DATE', None]] cnt = Counter(tags) tags = sorted(list(set([el for el in tags if cnt[el] > 1])), key = lambda t: cnt[t]*np.exp(-tags.index(t)))[::-1] tags = [" ".join(re.sub("[^A-Za-z0-9\s]", "", unidecode(tag)).split()) for tag in tags] tags = ['ᴛᴀɢ ' + el + '' for el in tags] tags = " ".join(tags) if tags: out_text = out_text + "

Tags: " + tags if warn_flag: out_text = out_text + "

Warning ⚠️: Unknown tokens detected in text. The model might behave erratically" return out_text init_text = '''The American Academy of Arts and Sciences (AAA&S) is one of the oldest learned societies in the United States. It was founded in 1780 during the American Revolution by John Adams, John Hancock, James Bowdoin, Andrew Oliver, and other Founding Fathers of the United States. It is headquartered in Cambridge, Massachusetts. Membership in the academy is achieved through a thorough petition, review, and election process. The academy's quarterly journal, Dædalus, is published by the MIT Press on behalf of the academy. The academy also conducts multidisciplinary public policy research. The Academy was established by the Massachusetts legislature on May 4, 1780, charted in order "to cultivate every art and science which may tend to advance the interest, honor, dignity, and happiness of a free, independent, and virtuous people." The sixty-two incorporating fellows represented varying interests and high standing in the political, professional, and commercial sectors of the state. The first class of new members, chosen by the Academy in 1781, included Benjamin Franklin and George Washington as well as several international honorary members. ''' init_output = extract(init_text) with gr.Blocks(css="footer {visibility: hidden}", theme=gr.themes.Default(text_size="lg", spacing_size="lg")) as interface: with gr.Row(): gr.Markdown(header) with gr.Row(): text = gr.Text(label="Extract entities", lines = 10, value = init_text) with gr.Row(): with gr.Column(): button = gr.Button("Extract").style(full_width=False) with gr.Row(): with gr.Column(): entities = gr.Markdown(init_output) with gr.Row(): with gr.Column(): gr.Markdown("
The input examples in this demo are extracted from https://it.wikipedia.org
") button.click(extract, inputs=[text], outputs = [entities]) interface.launch()