Upload 10 files

src/abbre_resolution.py

@@ -0,0 +1,434 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Tue Aug 11 16:52:40 2020
+
+@author: luol2
+"""
+
+import logging
+import regex
+import sys
+import io
+
+"""
+A Python 3 refactoring of Vincent Van Asch's Python 2 code at
+
+http://www.cnts.ua.ac.be/~vincent/scripts/abbreviations.py
+
+Based on
+
+A Simple Algorithm for Identifying Abbreviations Definitions in Biomedical Text
+A. Schwartz and M. Hearst
+Biocomputing, 2003, pp 451-462.
+
+"""
+
+logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)
+log = logging.getLogger('Abbre')
+
+
+class Candidate(str):
+    def __init__(self, value):
+        super().__init__()
+        self.start = 0
+        self.stop = 0
+
+    def set_position(self, start, stop):
+        self.start = start
+        self.stop = stop
+
+
+def yield_lines_from_file(file_path):
+    with open(file_path, 'rb') as f:
+        for line in f:
+            try:
+                line = line.decode('utf-8')
+            except UnicodeDecodeError:
+                line = line.decode('latin-1').encode('utf-8').decode('utf-8')
+            line = line.strip()
+            yield line
+        f.close()
+
+
+def yield_lines_from_doc(doc_text):
+    for line in doc_text.split("\n"):
+        yield line.strip()
+
+
+def best_candidates(sentence):
+    """
+    :param sentence: line read from input file
+    :return: a Candidate iterator
+    """
+
+    if '(' in sentence:
+        # Check some things first
+        if sentence.count('(') != sentence.count(')'):
+            raise ValueError("Unbalanced parentheses: {}".format(sentence))
+
+        if sentence.find('(') > sentence.find(')'):
+            raise ValueError("First parentheses is right: {}".format(sentence))
+
+        closeindex = -1
+        while 1:
+            # Look for open parenthesis
+            openindex = sentence.find('(', closeindex + 1)
+
+            if openindex == -1: break
+
+            # Look for closing parentheses
+            closeindex = openindex + 1
+            open = 1
+            skip = False
+            while open:
+                try:
+                    char = sentence[closeindex]
+                except IndexError:
+                    # We found an opening bracket but no associated closing bracket
+                    # Skip the opening bracket
+                    skip = True
+                    break
+                if char == '(':
+                    open += 1
+                elif char in [')', ';', ':']:
+                    open -= 1
+                closeindex += 1
+
+            if skip:
+                closeindex = openindex + 1
+                continue
+
+            # Output if conditions are met
+            start = openindex + 1
+            stop = closeindex - 1
+            candidate = sentence[start:stop]
+
+            # Take into account whitespace that should be removed
+            start = start + len(candidate) - len(candidate.lstrip())
+            stop = stop - len(candidate) + len(candidate.rstrip())
+            candidate = sentence[start:stop]
+
+            if conditions(candidate):
+                new_candidate = Candidate(candidate)
+                new_candidate.set_position(start, stop)
+                yield new_candidate
+
+
+def conditions(candidate):
+    """
+    Based on Schwartz&Hearst
+
+    2 <= len(str) <= 10
+    len(tokens) <= 2
+    re.search('\p{L}', str)
+    str[0].isalnum()
+
+    and extra:
+    if it matches (\p{L}\.?\s?){2,}
+    it is a good candidate.
+
+    :param candidate: candidate abbreviation
+    :return: True if this is a good candidate
+    """
+    viable = True
+    if regex.match('(\p{L}\.?\s?){2,}', candidate.lstrip()):
+        viable = True
+    if len(candidate) < 2 or len(candidate) > 10:
+        viable = False
+    if len(candidate.split()) > 2:
+        viable = False
+    if not regex.search('\p{L}', candidate):
+        viable = False
+    if not candidate[0].isalnum():
+        viable = False
+
+    return viable
+
+
+def get_definition(candidate, sentence):
+    """
+    Takes a candidate and a sentence and returns the definition candidate.
+
+    The definintion candidate is the set of tokens (in front of the candidate)
+    that starts with a token starting with the first character of the candidate
+
+    :param candidate: candidate abbreviation
+    :param sentence: current sentence (single line from input file)
+    :return: candidate definition for this abbreviation
+    """
+    # Take the tokens in front of the candidate
+    tokens = regex.split(r'[\s\-]+', sentence[:candidate.start - 2].lower())
+    #print(tokens)
+    # the char that we are looking for
+    key = candidate[0].lower()
+
+    # Count the number of tokens that start with the same character as the candidate
+#     print(tokens)
+    firstchars = [t[0] for t in tokens]
+#     print(firstchars)
+    definition_freq = firstchars.count(key)
+    candidate_freq = candidate.lower().count(key)
+
+    # Look for the list of tokens in front of candidate that
+    # have a sufficient number of tokens starting with key
+    if candidate_freq <= definition_freq:
+        # we should at least have a good number of starts
+        count = 0
+        start = 0
+        startindex = len(firstchars) - 1
+        
+        while count < candidate_freq:
+            if abs(start) > len(firstchars):
+                raise ValueError("candiate {} not found".format(candidate))
+            start -= 1
+            # Look up key in the definition
+            try:
+                startindex = firstchars.index(key, len(firstchars) + start)
+            except ValueError:
+                pass
+
+            # Count the number of keys in definition
+            count = firstchars[startindex:].count(key)
+        
+        # We found enough keys in the definition so return the definition as a definition candidate
+        start = len(' '.join(tokens[:startindex]))
+        stop = candidate.start - 1
+        candidate = sentence[start:stop]
+
+        # Remove whitespace
+        start = start + len(candidate) - len(candidate.lstrip())
+        stop = stop - len(candidate) + len(candidate.rstrip())
+        candidate = sentence[start:stop]
+
+        new_candidate = Candidate(candidate)
+        new_candidate.set_position(start, stop)
+        #print('new_candidate:')
+        #print(new_candidate,start,stop)
+        return new_candidate
+
+    else:
+        raise ValueError('There are less keys in the tokens in front of candidate than there are in the candidate')
+
+
+def select_definition(definition, abbrev):
+    """
+    Takes a definition candidate and an abbreviation candidate
+    and returns True if the chars in the abbreviation occur in the definition
+
+    Based on
+    A simple algorithm for identifying abbreviation definitions in biomedical texts, Schwartz & Hearst
+    :param definition: candidate definition
+    :param abbrev: candidate abbreviation
+    :return:
+    """
+
+
+    if len(definition) < len(abbrev):
+        raise ValueError('Abbreviation is longer than definition')
+
+    if abbrev in definition.split():
+        raise ValueError('Abbreviation is full word of definition')
+
+    sindex = -1
+    lindex = -1
+
+    while 1:
+        try:
+            longchar = definition[lindex].lower()
+        except IndexError:
+            raise
+
+        shortchar = abbrev[sindex].lower()
+
+        if not shortchar.isalnum():
+            sindex -= 1
+
+        if sindex == -1 * len(abbrev):
+            if shortchar == longchar:
+                if lindex == -1 * len(definition) or not definition[lindex - 1].isalnum():
+                    break
+                else:
+                    lindex -= 1
+            else:
+                lindex -= 1
+                if lindex == -1 * (len(definition) + 1):
+                    raise ValueError("definition {} was not found in {}".format(abbrev, definition))
+
+        else:
+            if shortchar == longchar:
+                sindex -= 1
+                lindex -= 1
+            else:
+                lindex -= 1
+#     print('lindex:',lindex,len(definition),definition[lindex:len(definition)])
+    new_candidate = Candidate(definition[lindex:len(definition)])
+    new_candidate.set_position(definition.start+lindex+len(definition), definition.stop)
+    definition = new_candidate
+
+    tokens = len(definition.split())
+    length = len(abbrev)
+
+    if tokens > min([length + 5, length * 2]):
+        raise ValueError("did not meet min(|A|+5, |A|*2) constraint")
+
+    # Do not return definitions that contain unbalanced parentheses
+    if definition.count('(') != definition.count(')'):
+        raise ValueError("Unbalanced parentheses not allowed in a definition")
+#     print('select:')
+#     print(definition,definition.start, definition.stop)
+    new_definition_dict={'definition':definition,'start':definition.start,'stop':definition.stop}
+    return new_definition_dict
+
+
+def extract_abbreviation_definition_pairs(file_path=None, doc_text=None):
+    abbrev_map = []
+    omit = 0
+    written = 0
+    if file_path:
+        sentence_iterator = enumerate(yield_lines_from_file(file_path))
+    elif doc_text:
+        sentence_iterator = enumerate(yield_lines_from_doc(doc_text))
+    else:
+        return abbrev_map
+
+    for i, sentence in sentence_iterator:
+        #print(sentence)
+        try:
+            for candidate in best_candidates(sentence):
+                #print(candidate)
+                try:
+                    #print('begin get definition')
+                    definition = get_definition(candidate, sentence)
+                    #print('get_definition:')
+                    #print(definition)
+                    
+                except (ValueError, IndexError) as e:
+                    #log.debug("{} Omitting candidate {}. Reason: {}".format(i, candidate, e.args[0]))
+                    omit += 1
+                else:
+                    try:
+                        definition_dict = select_definition(definition, candidate)
+                    except (ValueError, IndexError) as e:
+                        #log.debug("{} Omitting definition {} for candidate {}. Reason: {}".format(i, definition_dict, candidate, e.args[0]))
+                        omit += 1
+                    else:
+                        definition_dict['abbre']=candidate
+                        abbrev_map.append(definition_dict)
+                        written += 1
+        except (ValueError, IndexError) as e:
+            log.debug("{} Error processing sentence {}: {}".format(i, sentence, e.args[0]))
+    log.debug("{} abbreviations detected and kept ({} omitted)".format(written, omit))
+    return abbrev_map
+
+def postprocess_abbr(ner_result,ori_text):
+    
+    final_result={}
+    if len(ner_result)==0:
+        return []
+    # abbr recognition
+    abbr_result=extract_abbreviation_definition_pairs(doc_text=ori_text)
+    
+    # read ner results
+    nor_loc_list={} #{entity_name_location:entity_information}
+
+    for ele in ner_result:
+        nor_loc_list[str(ele[0])+' '+str(ele[1])]=ele
+        final_result['\t'.join(ele)]=[int(ele[0]),int(ele[1])]
+    
+    #abbr matching
+    for abbr in abbr_result:
+        abbr_index=str(abbr['start'])+' '+str(abbr['stop'])
+        if abbr_index in nor_loc_list.keys():
+
+            line=ori_text
+            abbr_text=abbr['abbre']
+            abbr_eid=0
+            while line.find(abbr_text)>=0:
+                abbr_sid=line.find(abbr_text)+abbr_eid
+                abbr_eid=abbr_sid+len(abbr_text)
+                # print(abbr_sid,abbr_eid)
+                if abbr_sid>0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False and ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+nor_loc_list[abbr_index][2]+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid==0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+nor_loc_list[abbr_index][2]+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid>0 and abbr_eid==len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False :
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+nor_loc_list[abbr_index][2]+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                line=ori_text[abbr_eid:]
+    # print(final_result)
+    sorted_final_result=sorted(final_result.items(), key=lambda kv:(kv[1]), reverse=False)
+    final_result=[]
+    for ele in sorted_final_result:
+        final_result.append(ele[0].split('\t'))
+    return final_result
+
+def ner_abbr(ner_result,abbr_result,ori_text):
+    # read ner results
+    nor_name_list={} #{entity_name:entity_information}
+    nor_loc_list={} #{entity_name_location:entity_information}
+    final_result={} #{entity_information:location}  use to sort
+    for ele in ner_result:
+        temp_seg=ele.split('\t')
+        nor_loc_list[temp_seg[0]+' '+temp_seg[1]]=temp_seg
+        nor_name_list[temp_seg[2].lower()]=temp_seg
+        final_result['\t'.join(temp_seg[0:4])]=[int(temp_seg[0]),int(temp_seg[1])]
+    
+    #abbr matching
+    for abbr in abbr_result:
+        abbr_index=str(abbr['start'])+' '+str(abbr['stop'])
+        if abbr_index in nor_loc_list.keys():
+
+            line=ori_text
+            abbr_text=abbr['abbre']
+            abbr_eid=0
+            while line.find(abbr_text)>=0:
+                abbr_sid=line.find(abbr_text)+abbr_eid
+                abbr_eid=abbr_sid+len(abbr_text)
+                # print(abbr_sid,abbr_eid)
+                if abbr_sid>0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False and ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+abbr_text+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid==0 and abbr_eid<len(ori_text):
+                    if ori_text[abbr_eid].isalnum()==False:
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+abbr_text+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                elif abbr_sid>0 and abbr_eid==len(ori_text):
+                    if ori_text[abbr_sid-1].isalnum()==False :
+                        final_result[str(abbr_sid)+'\t'+str(abbr_eid)+'\t'+abbr_text+'\t'+nor_loc_list[abbr_index][3]]=[abbr_sid,abbr_eid]
+                line=ori_text[abbr_eid:]
+    # print(final_result)
+    final_result=sorted(final_result.items(), key=lambda kv:(kv[1]), reverse=False)
+    
+    return final_result
+
+
+            
+            
+if __name__ == '__main__':
+    path='//panfs/pan1/bionlp/lulab/luoling/HPO_project/diseaseTag/data/test/results/'
+    fin=open(path+'NCBI_test_phecr_95.tsv','r',encoding='utf-8')
+    context=fin.read().strip().split('\n\n')
+    fin.close()
+    fout=open(path+'NCBI_test_phecr_abbre_95.tsv','w',encoding='utf-8')
+    for doc in context:
+        lines=doc.split('\n')
+        ori_text=lines[1]
+        # print(ori_text)
+        fout.write(lines[0]+'\n'+lines[1]+'\n')
+        if len(lines)>2:
+            abbr_result=extract_abbreviation_definition_pairs(doc_text=ori_text)
+            print(abbr_result)
+            abbr_out=ner_abbr(lines[2:],abbr_result,ori_text)
+        else:
+            abbr_out=[]
+        # print('final:',abbr_out)
+        for ele in abbr_out:
+            fout.write(ele[0]+'\n')
+        fout.write('\n')
+        # sys.exit()
+    fout.close()
+    #last_out=combine_ml_dict_fn(abbr_out,infile)
+    #print(last_out)
+    
+    

src/combine_result.py

@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Jun 15 11:24:45 2020
+
+@author: luol2
+"""
+
+import io
+def nest_overlap_entity(nest_list):
+    temp_result_list={}
+    for i in range(0, len(nest_list)):
+        hpoid=nest_list[i][3]
+        if hpoid not in temp_result_list.keys():
+            temp_result_list[hpoid]=nest_list[i]
+        else:
+            score=float(nest_list[i][4])
+            old_score=float(temp_result_list[hpoid][4])
+            if score>old_score: # retain higer score concept
+                temp_result_list[hpoid]=nest_list[i]
+    new_list=[]
+    for hpoid in temp_result_list.keys():
+        new_list.append(temp_result_list[hpoid])
+    
+    temp_result_list={} #same index, different ids
+    for i in range(0, len(new_list)):
+        ids=new_list[i][0]+' '+new_list[i][1]
+        if ids not in temp_result_list.keys():
+            temp_result_list[ids]=new_list[i]
+        else:
+            score=float(nest_list[i][4])
+            old_score=float(temp_result_list[ids][4])
+            if score>old_score: 
+                temp_result_list[ids]=new_list[i]
+    final_list=[]
+    for ids in temp_result_list.keys():
+        final_list.append(temp_result_list[ids]) 
+    return final_list
+def combine_ml_dict(dict_tsv,ml_tsv,nest=True):
+    fin_dic=io.StringIO(dict_tsv)
+    fin_ml=io.StringIO(ml_tsv)
+    fout=io.StringIO()
+    all_dic=fin_dic.read().strip().split('\n\n')
+    all_ml=fin_ml.read().strip().split('\n\n')
+    fin_dic.close()
+    fin_ml.close()
+    
+    for i in range(0,len(all_dic)):
+        lines_dic=all_dic[i].split('\n')
+        lines_ml=all_ml[i].split('\n')
+        entity_list={}
+        for j in range(1,len(lines_dic)):
+            seg=lines_dic[j].split('\t')
+            entity_list[lines_dic[j]]=[int(seg[0]),int(seg[1])] #dict results score 1.00
+        for j in range(1,len(lines_ml)):
+            seg=lines_ml[j].split('\t')
+            entity_list[lines_ml[j]]=[int(seg[0]),int(seg[1])]
+                
+        entity_list=sorted(entity_list.items(), key=lambda kv:(kv[1]), reverse=False)
+        entity_list_sort=[]
+        for ele in entity_list:
+            entity_list_sort.append(ele[0])
+        
+        final_entity=[]
+        if len(entity_list_sort)!=0:
+            first_entity=entity_list_sort[0].split('\t')
+            nest_list=[first_entity]
+            max_eid=int(first_entity[1])
+
+            for i in range(1,len(entity_list_sort)):
+                segs=entity_list_sort[i].split('\t')
+                if int(segs[0])> max_eid:
+                    if len(nest_list)==1:
+                        final_entity.append(nest_list[0])
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[1])>max_eid:
+                            max_eid=int(segs[1])
+                    else:
+                        tem=nest_overlap_entity(nest_list)
+                        final_entity.extend(tem)
+                        nest_list=[]
+                        nest_list.append(segs)
+                        if int(segs[1])>max_eid:
+                            max_eid=int(segs[1])
+                else:
+                    nest_list.append(segs)
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+            if nest_list!=[]:
+                if len(nest_list)==1:
+                    final_entity.append(nest_list[0])
+
+                else:
+                    tem=nest_overlap_entity(nest_list)#find max entity
+                    final_entity.extend(tem)
+        
+        fout.write(lines_ml[0]+'\n')
+        for ele in final_entity:
+            fout.write('\t'.join(ele)+'\n')
+        fout.write('\n')
+    return fout.getvalue()  
+                

src/ml_ner.py

@@ -0,0 +1,563 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 16:41:54 2020
+
+@author: luol2
+"""
+
+import io
+import time
+import numpy as np
+NEG_LABEL='ONT:None'
+def ml_intext(infile):
+    fin=open(infile,'r',encoding='utf-8')
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    data_list=[]
+    label_list=[]
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        label=lines[0].split('\t')[0]
+        label_list.append(label)
+        for i in range(1,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg)
+        data_list.append(temp_sentece)
+    return data_list,label_list
+def ml_intext_fn(ml_input):
+    fin=io.StringIO(ml_input)
+    alltexts=fin.read().strip().split('\n\n')
+    fin.close()
+    data_list=[]
+    label_list=[]
+    for sents in alltexts:
+        lines=sents.split('\n')
+        temp_sentece=[]
+        label=lines[0].split('\t')[0]
+        label_list.append(label)
+        for i in range(1,len(lines)):
+            seg=lines[i].split('\t')
+            temp_sentece.append(seg)
+        data_list.append(temp_sentece)
+    return data_list,label_list
+def pun_filter(temp_entity):
+    pun_list=[',','.','!',';',':','?','(',')','[',']','{','}']
+    filter_flag=0
+    for ele in temp_entity:
+        if ele in pun_list:
+            filter_flag=1
+            break
+    return filter_flag
+def pos_filter(temp_pos,temp_entity):
+    pos_list_l=['PRP']
+    pos_list=['IN','DT','CC','O','MD','EX','POS','WDT','WP','WP$','WRB','TO','PRP$']
+    verb_word=['is','are','was','were','had','have','has','be','been','also']
+    filter_flag=0
+
+    if (temp_entity[0] in verb_word) or (temp_entity[-1] in verb_word):
+        filter_flag=1
+    if (temp_pos[0] in pos_list) or (temp_pos[-1] in pos_list) or (temp_pos[0] in pos_list_l):
+        filter_flag=1
+    return filter_flag 
+    
+def build_ngram_testset_filted(conll_input,Ngram=8):
+
+    fin_genia=io.StringIO(conll_input)
+    fout_context=io.StringIO()
+    fout_txt=io.StringIO()
+   
+    index_dict={}
+    allentity=[]
+    alltext=fin_genia.read().strip().split('\n\n')
+    fin_genia.close()
+    num_total=0
+    for i in range(0,len(alltext)):
+        
+        lines=alltext[i].split('\n') 
+        ori_txt=[]
+        for ele in lines:
+            seg=ele.split('\t')
+            ori_txt.append(seg[0])
+        fout_txt.write(' '.join(ori_txt)+'\n')
+
+        if Ngram>len(lines):
+            Ngram=len(lines)
+        
+        fout_context_list=[]
+        temp_entity=[]
+        temp_pos=[]
+        for ngram in range(2,Ngram+1):
+            if ngram==1:
+                for j in range(0, len(lines)):                 
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            sid=m
+                            eid=m
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])                            
+                        else:
+                            pass
+    #                        print(sentence[m])
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write(NEG_LABEL+'\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            elif ngram==2:
+                for j in range(0, len(lines)-1):                  
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m==j+1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write(NEG_LABEL+'\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            else :
+                for j in range(0, len(lines)-ngram+1):               
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m>j and m<j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[2])
+                        elif m==j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write(NEG_LABEL+'\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+
+    return fout_context.getvalue(),fout_txt.getvalue(),index_dict
+
+def build_all_ngram_testset_filted(conll_input,Ngram=8):
+
+    fin_genia=io.StringIO(conll_input)
+    fout_context=io.StringIO()
+    fout_txt=io.StringIO()
+   
+    index_dict={}
+    allentity=[]
+    alltext=fin_genia.read().strip().split('\n\n')
+    fin_genia.close()
+    num_total=0
+    for i in range(0,len(alltext)):
+        
+        lines=alltext[i].split('\n') 
+        ori_txt=[]
+        for ele in lines:
+            seg=ele.split('\t')
+            ori_txt.append(seg[0])
+        fout_txt.write(' '.join(ori_txt)+'\n')
+
+        if Ngram>len(lines):
+            Ngram=len(lines)
+        
+        fout_context_list=[]
+        temp_entity=[]
+        temp_pos=[]
+        for ngram in range(1,Ngram+1):
+            if ngram==1:
+                for j in range(0, len(lines)):                 
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            sid=m
+                            eid=m
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])                            
+                        else:
+                            pass
+    #                        print(sentence[m])
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write(NEG_LABEL+'\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            elif ngram==2:
+                for j in range(0, len(lines)-1):                  
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m==j+1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write(NEG_LABEL+'\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+            else :
+                for j in range(0, len(lines)-ngram+1):               
+                    sid=0
+                    eid=0
+                    for m in range(0,len(lines)):
+                        if m==j:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            sid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        elif m>j and m<j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[2])
+                        elif m==j+ngram-1:
+                            fout_context_list.append(lines[m]+'\tO\tB')
+                            eid=m
+                            temp_seg=lines[m].split('\t')
+                            temp_entity.append(temp_seg[0])
+                            temp_pos.append(temp_seg[3])
+                        else:
+                            pass
+#                            fout_context_list.append(lines[m]+'\tO\tO')
+                    
+                    if pun_filter(temp_entity)==0 and pos_filter(temp_pos,temp_entity)==0:
+                        num_total+=1
+                        if ' '.join(temp_entity) not in allentity:
+                            allentity.append(' '.join(temp_entity))
+                        fout_context.write(NEG_LABEL+'\t'+' '.join(temp_entity)+'\n')
+                        fout_context.write('\n'.join(fout_context_list)+'\n\n')
+                        index_dict[str(num_total)]=[i,sid,eid]
+
+                    temp_entity=[]
+                    temp_pos=[]
+                    fout_context_list=[]
+
+    return fout_context.getvalue(),fout_txt.getvalue(),index_dict
+
+def output_result(result,label_2_index,Top_N=5):
+
+    fout=io.StringIO()
+    ont_label={}
+
+    for key in label_2_index.keys():
+        ont_label[label_2_index[key]]=key
+
+
+    for line in result:
+        #Top_index=line.argsort()[-1*Top_N:][::-1]
+        index_top_unsort=np.argpartition(line,-Top_N)[-Top_N:]
+        values_top=line[index_top_unsort]
+        Top_index=index_top_unsort[np.argsort(-values_top)]
+        temp_list=[]
+        for max_index in Top_index:
+            ont_id=ont_label[max_index]
+            ont_id_value=round(line[max_index],5)
+            temp_list.append(str(ont_id)+'|'+str(ont_id_value))
+        fout.write('\t'.join(temp_list)+'\n')
+        
+    return fout.getvalue()
+
+def decode_tsv(test_score, ml_input_index, ml_input_txt, T=0.8):
+    
+    fin_predict=io.StringIO(test_score)
+    fin_text=io.StringIO(ml_input_txt)
+    fout=io.StringIO()
+    
+    test_txt=fin_text.read().strip().split('\n')
+    test_index=ml_input_index
+    test_pre=fin_predict.read().strip().split('\n')
+    
+    fin_text.close()
+    fin_predict.close()
+    
+    sent_result={}
+    for i in range(0,len(test_pre)):
+        seg_pre=test_pre[i].split('\t')[0].split('|')
+        #print(seg_pre,T)
+        if float(seg_pre[1])>T and seg_pre[0]!=NEG_LABEL:
+            term_id=str(i+1)
+            pre_result=[test_index[term_id][1],test_index[term_id][2],seg_pre[0],seg_pre[1]]
+            sent_id=str(test_index[term_id][0])
+            if sent_id not in sent_result.keys():
+                sent_result[sent_id]=[pre_result]
+            else:
+                sent_result[sent_id].append(pre_result)
+        
+    for i in range(0,len(test_txt)):
+        fout.write(test_txt[i]+'\n')
+        if str(i) in sent_result.keys():
+            temp_result={}
+            for ele in sent_result[str(i)]:
+                temp_line=str(ele[0])+'\t'+str(ele[1])+'\t'+' '.join(test_txt[i].split()[ele[0]:ele[1]+1])+'\t'+ele[2]+'\t'+ele[3]
+                temp_result[temp_line]=[ele[0],ele[1]]
+            if len(temp_result)>=1:
+                temp_result=sorted(temp_result.items(), key=lambda d: (d[1][0],d[1][1]), reverse=False)
+                for ent in temp_result:
+                    fout.write(ent[0]+'\n')
+        fout.write('\n')
+
+    return fout.getvalue()
+    
+def score_filter(temp_entity,  T=0.1):
+
+    result_list=[]
+    for i in range(0,len(temp_entity)):
+        if float (temp_entity[i][-1])>=T:
+            result_list.append(temp_entity[i])
+    return(result_list)
+def find_max_entity_nest(nest_list):
+    temp_result_list={}
+    for i in range(0, len(nest_list)):
+        hpoid=nest_list[i][-2]
+        score=float(nest_list[i][-1])
+        if hpoid not in temp_result_list.keys():
+            temp_result_list[hpoid]=nest_list[i]
+        else:
+            if score>float(temp_result_list[hpoid][-1]):
+                temp_result_list[hpoid]=nest_list[i]
+    new_list=[]
+    for hpoid in temp_result_list.keys():
+        new_list.append(temp_result_list[hpoid])
+    return new_list
+def duplicate_filter(temp_entity):
+    result_list=[]
+    if len(temp_entity)>1:                   
+        first_entity=temp_entity[0]
+        nest_list=[first_entity]
+        max_eid=int(first_entity[1])
+    
+        for i in range(1,len(temp_entity)):
+            segs=temp_entity[i]
+            if int(segs[0])> max_eid:
+                if len(nest_list)==1:
+                    result_list.append(nest_list[0])
+                    nest_list=[segs]
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                else:
+                    result_list.extend(find_max_entity_nest(nest_list))
+                    nest_list=[segs]
+
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                    
+            else:
+                nest_list.append(segs)
+                if int(segs[1])>max_eid:
+                    max_eid=int(segs[1])
+        if nest_list!=[]:
+            if len(nest_list)==1:
+                result_list.append(nest_list[0])
+
+            else:
+                result_list.extend(find_max_entity_nest(nest_list))
+    else:
+        result_list=temp_entity
+    return result_list
+def combine_strategy(test_decode_temp, T=0.8):
+    fin=io.StringIO(test_decode_temp)
+    fout=io.StringIO()
+    
+    documents=fin.read().strip().split('\n\n')
+    fin.close()
+    
+    for doc in documents:
+        lines=doc.split('\n')
+        context=lines[0]
+        final_entity_list=[]
+        if len(lines)>1:
+            # all entity candidates
+            temp_entity=[]
+            for i in range(1,len(lines)):
+                temp_entity.append(lines[i].split('\t'))
+            #print('all entity condidates: ',len(temp_entity))
+            
+            # 将阈值低于T的候选过滤
+            filter1=score_filter(temp_entity,T)
+#            print('filter1:', len(filter1))
+            filter2=duplicate_filter(filter1)  
+            #print('filter2:', filter2)
+            final_entity_list=filter2
+            
+        fout.write(context+'\n')
+        for ele in final_entity_list:
+            fout.write('\t'.join(ele)+'\n')
+        fout.write('\n')
+    
+    return fout.getvalue() 
+
+
+def model_predict(ml_input,nn_model,ml_input_txt,ml_input_index,Threshold):
+    if nn_model.model_type=='cnn':
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x, test_y = nn_model.rep.represent_instances_all_feas(test_set,test_label,word_max_len=nn_model.hyper['sen_max'],char_max_len=nn_model.hyper['word_max'])
+        input_test = []
+    
+        if nn_model.fea_dict['word'] == 1:
+            input_test.append(test_x[0])
+    
+        if nn_model.fea_dict['char'] == 1:
+            input_test.append(test_x[1])
+    
+        if nn_model.fea_dict['lemma'] == 1:
+            input_test.append(test_x[2])
+    
+        if nn_model.fea_dict['pos'] == 1:
+            input_test.append(test_x[3])
+    
+        test_pre = nn_model.model.predict(input_test,batch_size=256,verbose=0)
+    
+    elif nn_model.model_type=='bert':
+
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x,test_y=nn_model.rep.load_data(test_set,test_label,word_max_len=nn_model.maxlen)
+        test_pre = nn_model.model.predict(test_x,batch_size=128,verbose=0)
+    
+    test_score=output_result(test_pre, nn_model.rep.label_2_index,Top_N=3)
+    #print('test_score:',test_score)
+    test_decode_temp=decode_tsv(test_score, ml_input_index, ml_input_txt,  T=Threshold)
+    #print('decode_temp:\n',test_decode_temp)
+    # test_pre_tsv=combine_strategy(test_decode_temp,T=Threshold) 
+    return test_decode_temp
+
+def model_predict_old(ml_input,nn_model,ml_input_txt,ml_input_index,Threshold):
+    if nn_model.model_type=='cnn':
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x, test_y = nn_model.rep.represent_instances_all_feas(test_set,test_label,word_max_len=nn_model.hyper['sen_max'],char_max_len=nn_model.hyper['word_max'])
+        input_test = []
+
+        if nn_model.fea_dict['word'] == 1:
+            input_test.append(test_x[0])
+
+        if nn_model.fea_dict['char'] == 1:
+            input_test.append(test_x[1])
+
+        if nn_model.fea_dict['lemma'] == 1:
+            input_test.append(test_x[2])
+
+        if nn_model.fea_dict['pos'] == 1:
+            input_test.append(test_x[3])
+
+        test_pre = nn_model.model.predict(input_test,batch_size=256)
+
+    elif nn_model.model_type=='bert':
+
+        test_set,test_label = ml_intext_fn(ml_input)
+        test_x,test_y=nn_model.rep.load_data(test_set,test_label,word_max_len=nn_model.maxlen)
+        test_pre = nn_model.model.predict(test_x,batch_size=128)
+
+    test_score=output_result(test_pre, nn_model.rep.label_2_index,Top_N=3)
+    #print('test_score:',test_score)
+    test_decode_temp=decode_tsv(test_score, ml_input_index, ml_input_txt,  T=0.0)
+    #print('decode_temp:\n',test_decode_temp)
+    test_pre_tsv=combine_strategy(test_decode_temp,T=Threshold) 
+    return test_pre_tsv
+
+def output_txt(ml_input_txt):
+    fin_text=io.StringIO(ml_input_txt)
+    fout=io.StringIO()
+
+    test_txt=fin_text.read().strip().split('\n')
+
+    fin_text.close()
+
+    for i in range(0,len(test_txt)):
+        fout.write(test_txt[i]+'\n')
+        fout.write('\n')
+
+    return fout.getvalue()
+
+def ml_tagging(ssplit_token,ml_model,Threshold):
+    ml_input, ml_input_txt,ml_input_index=build_ngram_testset_filted(ssplit_token)
+    #print('ml_input:')
+    #print(ml_input)
+    if len(ml_input_index)>0:
+        ml_pre_tsv=model_predict(ml_input,ml_model,ml_input_txt,ml_input_index,Threshold)
+    else:
+        ml_pre_tsv=output_txt(ml_input_txt)
+    return ml_pre_tsv
+    
+def ml_tagging_allngram(ssplit_token,ml_model,Threshold):
+    ml_input, ml_input_txt,ml_input_index=build_all_ngram_testset_filted(ssplit_token)
+    #print('ml_input:')
+    #print(ml_input)
+    if len(ml_input_index)>0:
+        ml_pre_tsv=model_predict_old(ml_input,ml_model,ml_input_txt,ml_input_index,Threshold)
+    else:
+        ml_pre_tsv=output_txt(ml_input_txt)
+    return ml_pre_tsv

src/nn_model.py

@@ -0,0 +1,130 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Mar 26 09:04:13 2020
+
+@author: luol2
+"""
+
+import time
+import sys
+import numpy as np
+import tensorflow as tf
+from src.nn_represent import CNN_RepresentationLayer,BERT_RepresentationLayer
+from tensorflow.keras.layers import *
+from tensorflow.keras.models import Model
+# from keras_bert import load_trained_model_from_checkpoint
+from transformers import TFAutoModel
+
+
+'''
+import keras.backend.tensorflow_backend as KTF
+physical_devices =tf.config.experimental.list_physical_devices('GPU')
+'''
+
+class bioTag_CNN():
+    def __init__(self, model_files):
+        self.model_type='cnn'
+        model_test_type='cnn'
+        self.fea_dict = {'word': 1,
+                         'char': 1,
+                         'lemma':0,
+                         'pos':0}
+    
+        self.hyper = {'sen_max'      :20,
+                      'word_max'     :40,
+                      'charvec_size' :50,
+                      'pos_size'     :50}
+             
+        self.w2vfile=model_files['w2vfile']      
+        self.charfile=model_files['charfile']
+        self.labelfile=model_files['labelfile']
+        self.posfile=model_files['posfile']
+          
+        vocab={'char':self.charfile,'label':self.labelfile,'pos':self.posfile}
+        print('loading w2v model.....') 
+        self.rep = CNN_RepresentationLayer(self.w2vfile,vocab_file=vocab, frequency=400000)
+         
+        print('building  model......')
+        all_fea = []
+        fea_list = []
+        
+        if self.fea_dict['word'] == 1:
+            word_input = Input(shape=(self.hyper['sen_max'],), dtype='int32', name='word_input')  
+            all_fea.append(word_input)
+            word_fea = Embedding(self.rep.vec_table.shape[0], self.rep.vec_table.shape[1], weights=[self.rep.vec_table], trainable=True,mask_zero=False, input_length=self.hyper['sen_max'], name='word_emd')(word_input)
+            fea_list.append(word_fea)
+    
+        if self.fea_dict['char'] == 1:
+            char_input = Input(shape=(self.hyper['sen_max'],self.hyper['word_max']), dtype='int32', name='char_input')
+            all_fea.append(char_input)
+            char_fea = TimeDistributed(Embedding(self.rep.char_table_size, self.hyper['charvec_size'], trainable=True,mask_zero=False),  name='char_emd')(char_input)
+            char_fea = TimeDistributed(Conv1D(self.hyper['charvec_size']*2, 3, padding='same',activation='relu'), name="char_cnn")(char_fea)
+            char_fea_max = TimeDistributed(GlobalMaxPooling1D(), name="char_pooling_max")(char_fea)
+            fea_list.append(char_fea_max)
+            
+        if self.fea_dict['lemma'] == 1:
+            lemma_input = Input(shape=(self.hyper['sen_max'],), dtype='int32', name='lemma_input')  
+            all_fea.append(lemma_input)
+            lemma_fea = Embedding(self.rep.vec_table.shape[0], self.rep.vec_table.shape[1], weights=[self.rep.vec_table], trainable=True,mask_zero=False, input_length=self.hyper['sen_max'], name='lemma_emd')(lemma_input)
+            fea_list.append(lemma_fea)
+            
+        if self.fea_dict['pos'] == 1:
+            pos_input = Input(shape=(self.hyper['sen_max'],), dtype='int32', name='pos_input')
+            all_fea.append(pos_input)
+            pos_fea = Embedding(self.rep.pos_table_size, self.hyper['pos_size'], trainable=True,mask_zero=False, input_length=self.hyper['sen_max'], name='pos_emd')(pos_input)
+            fea_list.append(pos_fea)    
+    
+        if len(fea_list) == 1:
+            concate_vec = fea_list[0]
+        else:
+            concate_vec = Concatenate()(fea_list)
+    
+        concate_vec = Dropout(0.4)(concate_vec)
+    
+        # model
+        if model_test_type=='cnn':    
+            cnn = Conv1D(1024, 1, padding='valid', activation='relu',name='cnn1')(concate_vec)
+            cnn = GlobalMaxPooling1D()(cnn)
+        elif model_test_type=='lstm':
+            bilstm = Bidirectional(LSTM(200, return_sequences=True, implementation=2, dropout=0.4, recurrent_dropout=0.4), name='bilstm1')(concate_vec)
+            cnn = GlobalMaxPooling1D()(bilstm)
+
+    
+        dense = Dense(1024, activation='relu')(cnn)
+        dense= Dropout(0.4)(dense)
+        output = Dense(self.rep.label_table_size, activation='softmax')(dense)
+        self.model = Model(inputs=all_fea, outputs=output)
+    def load_model(self,model_file):
+        self.model.load_weights(model_file)
+        self.model.summary()        
+        print('load model done!')
+
+class bioTag_BERT():
+    def __init__(self, model_files):
+        self.model_type='bert'
+        self.maxlen = 32
+        
+        self.checkpoint_path = model_files['checkpoint_path']
+        self.label_file=model_files['labelfile']
+        self.lowercase=model_files['lowercase']
+        self.rep = BERT_RepresentationLayer(self.checkpoint_path, self.label_file, lowercase=self.lowercase)
+       
+        
+        plm_model = TFAutoModel.from_pretrained(self.checkpoint_path, from_pt=True)
+
+        x1_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='input_ids')
+        x2_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='token_type_ids')
+        x3_in = Input(shape=(self.maxlen,),dtype=tf.int32, name='attention_mask')
+        #x = plm_model(x1_in, token_type_ids=x2_in, attention_mask=x3_in)[1]
+        #x = plm_model(x1_in, token_type_ids=x2_in, attention_mask=x3_in)[0] 
+        #x = GlobalMaxPooling1D()(x)
+        x = plm_model(x1_in, token_type_ids=x2_in, attention_mask=x3_in)[0][:,0,:] #[CLS] embedding
+        outputs = Dense(self.rep.label_table_size, activation='softmax')(x)
+    
+        self.model = Model(inputs=[x1_in,x2_in,x3_in], outputs=outputs)
+
+    def load_model(self,model_file):
+        self.model.load_weights(model_file)
+        self.model.summary()        
+
+      

src/nn_represent.py

@@ -0,0 +1,289 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 10:02:20 2020
+
+@author: luol2
+"""
+import time
+import os, sys
+import numpy as np
+from tensorflow.keras.preprocessing.sequence import pad_sequences
+# from keras_bert import Tokenizer
+from transformers import AutoTokenizer
+
+
+class CNN_RepresentationLayer(object):
+    
+    
+    def __init__(self, wordvec_file,  vocab_file=[],\
+                 vec_size=50, word_size=10000, frequency=10000):
+        
+        '''
+        wordvec_file    ：    the file path of word embedding
+        vec_size        :    the dimension size of word vector 
+                             learned by word2vec tool
+        
+        word_size       :    the size of word vocabulary
+  
+        frequency       :    the threshold for the words left according to
+                             their frequency appeared in the text
+                             for example, when frequency is 10000, the most
+                             frequent appeared 10000 words are considered
+        
+        '''
+        #load word embedding
+        file = open(wordvec_file)
+        first_line = file.readline().strip()
+        file.close()
+        self.word_size = int(first_line.split()[0])
+        self.vec_size = int(first_line.split()[1])
+        self.frequency = frequency
+        
+        if self.frequency>self.word_size:
+            self.vec_table = np.zeros((self.word_size + 2, self.vec_size))
+        else:
+            self.vec_table = np.zeros((self.frequency + 2, self.vec_size))
+        self.word_2_index = {}
+        self.load_wordvecs(wordvec_file)
+        
+        #other fea
+        self.char_2_index={}
+        self.char_table_size=0
+        if 'char' in vocab_file.keys():
+            self.load_fea_vocab(vocab_file['char'],self.char_2_index)
+            self.char_table_size=len(self.char_2_index)
+            #print(self.char_table_size)
+            #print(self.char_2_index)
+        
+        self.label_2_index={}
+        self.label_table_size=0
+        if 'label' in vocab_file.keys():
+            self.load_label_vocab(vocab_file['label'],self.label_2_index)
+            self.label_table_size=len(self.label_2_index)
+            #print(self.label_table_size)
+            #print(self.char_2_index) 
+                        
+        self.pos_2_index={}
+        self.pos_table_size=0
+        if 'pos' in vocab_file.keys():
+            self.load_fea_vocab(vocab_file['pos'],self.pos_2_index)
+            self.pos_table_size=len(self.pos_2_index)
+            #print(self.pos_table_size)
+            
+
+    
+    def load_wordvecs(self, wordvec_file):
+        
+        file = open(wordvec_file,'r',encoding='utf-8')
+        file.readline()
+        #print(self.word_size)
+        #print(self.vec_size)
+        row = 0
+        self.word_2_index['padding_0'] = row #oov-zero vector
+        row+=1
+        for line in file:
+            if row <= self.word_size and row <= self.frequency:
+                line_split = line.strip().split(' ')
+                self.word_2_index[line_split[0]] = row
+                for col in range(self.vec_size):
+                    self.vec_table[row][col] = float(line_split[col + 1])
+                row += 1
+            else:
+                break
+        
+        self.word_2_index['sparse_vectors'] = row #oov-zero vector      
+        file.close()        
+
+    def load_fea_vocab(self,fea_file,fea_index):
+        fin=open(fea_file,'r',encoding='utf-8')
+        i=0
+        fea_index['padding_0']=i
+        i+=1
+        fea_index['oov_padding']=i
+        i+=1
+        for line in fin:
+            fea_index[line.strip()]=i
+            i+=1
+        fin.close()
+        
+    def load_label_vocab(self,fea_file,fea_index):
+        fin=open(fea_file,'r',encoding='utf-8')
+        i=0
+        for line in fin:
+            fea_index[line.strip()]=i
+            i+=1
+        fin.close()
+    
+    '''
+    def generate_label_list(self,labels):
+        label_list=[]
+        
+        for label in labels:
+            temp_label=[0]*self.label_table_size
+            temp_label[self.label_2_index[label]]=1
+            label_list.append(temp_label)
+        return label_list
+    '''
+    def generate_label_list(self,labels):
+        sparse_labels=[]
+        for ele in labels:
+            sparse_labels.append(self.label_2_index[ele])
+        return(sparse_labels)
+    
+    def represent_instances_all_feas(self, instances, labels, word_max_len=100, char_max_len=50, training=False):
+
+        x_text_list=[]
+        x_word_list=[]
+        x_char_list=[]
+        x_lemma_list=[]
+        x_pos_list=[]
+
+        y_list=[]
+
+        for sentence in instances:           
+            sentence_list=[]
+            sentence_word_list=[]
+            sentence_lemma_list=[]
+            sentence_pos_list=[]
+            sentence_text=[]   
+            for j in range(0,len(sentence)):
+                word=sentence[j]
+                #char fea               
+                char_list=[0]*char_max_len
+                for i in range(len(word[0])):
+                    if i<char_max_len:
+                        if word[0][i] in self.char_2_index.keys():
+                            char_list[i]=self.char_2_index[word[0][i]]
+                        else:
+                            char_list[i]=self.char_2_index['oov_padding']
+                sentence_word_list.append(char_list)
+                
+                #word fea
+                sentence_text.append(word[0].lower())
+                if word[0].lower() in self.word_2_index.keys():
+                    sentence_list.append(self.word_2_index[word[0].lower()])                        
+                else:
+                    sentence_list.append(self.word_2_index['sparse_vectors'])
+                
+                #lemma fea
+                if word[1].lower() in self.word_2_index.keys():
+                    sentence_lemma_list.append(self.word_2_index[word[1].lower()])
+                else:
+                    sentence_lemma_list.append(self.word_2_index['sparse_vectors'])
+             
+                #pos fea
+                if word[3] in self.pos_2_index.keys():
+                    sentence_pos_list.append(self.pos_2_index[word[3]])
+                else:
+                    sentence_pos_list.append(self.pos_2_index['oov_padding'])
+                
+            x_text_list.append(sentence_text)
+            x_word_list.append(sentence_list)
+            x_char_list.append(sentence_word_list)
+            x_lemma_list.append(sentence_lemma_list)
+            x_pos_list.append(sentence_pos_list)
+        
+        if training==True:
+            y_list=self.generate_label_list(labels)
+            x_word_np = pad_sequences(x_word_list, word_max_len, value=0, padding='post',truncating='post')  # right padding
+            x_char_np = pad_sequences(x_char_list, word_max_len, value=0, padding='post',truncating='post')
+            x_lemma_np = pad_sequences(x_lemma_list, word_max_len, value=0, padding='post',truncating='post')                
+            x_pos_np = pad_sequences(x_pos_list, word_max_len, value=0, padding='post',truncating='post')
+            y_np = np.array(y_list)
+            
+        else:
+            x_word_np = pad_sequences(x_word_list, word_max_len, value=0, padding='post',truncating='post')  # right padding
+            x_char_np = pad_sequences(x_char_list, word_max_len, value=0, padding='post',truncating='post')
+            x_lemma_np=[]
+            x_pos_np=[]
+            y_np=[]
+            
+        return [x_word_np, x_char_np, x_lemma_np,  x_pos_np, x_text_list], y_np        
+
+
+
+class BERT_RepresentationLayer(object):
+    
+    
+    def __init__(self, tokenizer_name_or_path, label_file,lowercase=True):
+        
+
+        #load vocab
+        self.model_type='bert'
+        #self.model_type='roberta'
+        if self.model_type in {"gpt2", "roberta"}:
+            self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True, add_prefix_space=True,do_lower_case=lowercase)
+        else:
+            self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_name_or_path, use_fast=True,do_lower_case=lowercase)
+
+        #load label
+        self.label_2_index={}
+        self.index_2_label={}
+        self.label_table_size=0
+        self.load_label_vocab(label_file,self.label_2_index,self.index_2_label)
+        self.label_table_size=len(self.label_2_index)
+        self.vocab_len=len(self.tokenizer)
+
+    def load_label_vocab(self,fea_file,fea_index,index_2_label):
+        
+        fin=open(fea_file,'r',encoding='utf-8')
+        all_text=fin.read().strip().split('\n')
+        fin.close()
+        for i in range(0,len(all_text)):
+            fea_index[all_text[i]]=i
+            index_2_label[str(i)]=all_text[i]
+            
+    def generate_label_list(self,labels):
+        sparse_labels=[]
+        for ele in labels:
+            sparse_labels.append(self.label_2_index[ele])       
+        return(sparse_labels)
+    
+    def load_data(self,instances, labels,  word_max_len=100,training=False):
+    
+        x_index=[]
+        x_seg=[]
+        x_mask=[]
+        y_list=[]
+        
+        for sentence in instances:                           
+            sentence_text_list=[]
+            for j in range(0,len(sentence)):
+                sentence_text_list.append(sentence[j][0].lower()) #input lower
+                
+            token_result=self.tokenizer(
+                sentence_text_list,
+                max_length=word_max_len,
+                truncation=True,is_split_into_words=True)
+            
+            bert_tokens=self.tokenizer.convert_ids_to_tokens(token_result['input_ids'])
+            word_index=token_result.word_ids(batch_index=0)
+            
+
+            x_index.append(token_result['input_ids'])
+            if self.model_type in {"gpt2", "roberta"}:
+                x_seg.append([0]*len(token_result['input_ids']))
+            else:
+                x_seg.append(token_result['token_type_ids'])
+            x_mask.append(token_result['attention_mask'])              
+        
+        if training==True:
+            y_list=self.generate_label_list(labels)
+        
+            x1_np = pad_sequences(x_index, word_max_len, value=0, padding='post',truncating='post')  # right padding
+            x2_np = pad_sequences(x_seg, word_max_len, value=0, padding='post',truncating='post')
+            x3_np = pad_sequences(x_mask, word_max_len, value=0, padding='post',truncating='post')
+            y_np = np.array(y_list)
+        
+        else:
+            x1_np = pad_sequences(x_index, word_max_len, value=0, padding='post',truncating='post')  # right padding
+            x2_np = pad_sequences(x_seg, word_max_len, value=0, padding='post',truncating='post')
+            x3_np = pad_sequences(x_mask, word_max_len, value=0, padding='post',truncating='post')
+            y_np=[]
+
+        return [x1_np, x2_np, x3_np], y_np  
+
+if __name__ == '__main__':
+    pass
+ 
+            

src/post_processing.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Jun 18 20:08:30 2020
+
+@author: luol2
+"""
+
+def combine_overlap(mention_list):
+   
+    entity_list=[]
+    if len(mention_list)>2:
+        
+        first_entity=mention_list[0]
+        nest_list=[first_entity]
+        max_eid=int(first_entity[1])
+        for i in range(1,len(mention_list)):
+            segs=mention_list[i]
+            if int(segs[0])> max_eid:
+                if len(nest_list)==1:
+                    entity_list.append(nest_list[0])
+                    nest_list=[]
+                    nest_list.append(segs)
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                else:
+                    tem=find_max_entity(nest_list)#find max entity
+                    entity_list.append(tem)
+                    nest_list=[]
+                    nest_list.append(segs)
+                    if int(segs[1])>max_eid:
+                        max_eid=int(segs[1])
+                    
+            else:
+                nest_list.append(segs)
+                if int(segs[1])>max_eid:
+                    max_eid=int(segs[1])
+        if nest_list!=[]:
+            if len(nest_list)==1:
+                entity_list.append(nest_list[0])
+
+            else:
+                tem=find_max_entity(nest_list)#find max entity
+                entity_list.append(tem)
+    else:
+        entity_list=mention_list
+        
+    return entity_list
+
+def find_max_entity(nest_list):
+    max_len=0
+    max_entity=[]
+    for i in range(0, len(nest_list)):
+        length=int(nest_list[i][1])-int(nest_list[i][0])
+        if length>max_len:
+                max_len=length
+                max_entity=nest_list[i]
+    
+    return max_entity

src/restore_index.py

@@ -0,0 +1,109 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Sun Jun 14 17:19:02 2020
+
+@author: luol2
+"""
+
+import io
+import sys
+        
+def restore_index_nest_fn(ori_text,file_pre):
+
+
+    fin_pre=io.StringIO(file_pre)
+    #print(file_pre)
+    all_pre=fin_pre.read().strip().split('\n\n')
+    fin_pre.close()
+    #print(len(all_pre))
+    
+    new_sentence=''
+    restore_result=[]
+    
+    sentence_ori=ori_text.lower().replace('``','" ')
+    sentence_ori=sentence_ori.replace("''",'" ')
+    for pre_i in range(0,len(all_pre)):
+        pre_lines=all_pre[pre_i].split('\n')
+        #print(pre_lines)
+#        print(sentence_ori)
+        if len(pre_lines)>1:
+            #print(pre_lines)
+            sentence_pre=pre_lines[0].lower().replace('``','"')
+            sentence_pre=sentence_pre.replace("''",'"')
+            sentence_pre=sentence_pre.split()
+            pre_result=[]
+            for i in range(1,len(pre_lines)):
+                pre_result.append(pre_lines[i].split('\t'))
+            
+            restore_sid=0
+            restore_eid=0
+            each_word_id=[]
+            
+            for i in range(0,len(sentence_pre)):
+
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                    if sentence_pre[i].find('"')>=0:
+                        temp_id = sentence_ori.find(sentence_pre[i].replace('"','" '))
+                    else:
+                        #print('ori:',sentence_ori)
+                        print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                
+                restore_sid=len(new_sentence)
+                restore_eid=len(new_sentence)+len(sentence_pre[i])
+                each_word_id.append([str(restore_sid),str(restore_eid)])
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+#            print('each_word:',each_word_id)    
+            for pre_ele in pre_result:
+                # if len(pre_ele)>4:
+                #     temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[3].split('|')[0],pre_ele[4]]
+                # else:
+                #     temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[3].split('|')[0],'1.00']
+                temp_pre_result=[each_word_id[int(pre_ele[0])][0],each_word_id[int(pre_ele[1])][1],pre_ele[3].split('|')[0],pre_ele[4]]
+                if temp_pre_result not in restore_result:
+                    restore_result.append(temp_pre_result)
+        else:
+            sentence_pre=pre_lines[0].lower().replace('``','"')
+            sentence_pre=sentence_pre.replace("''",'"')
+            sentence_pre=sentence_pre.split()
+           
+            for i in range(0,len(sentence_pre)):
+
+                temp_id=sentence_ori.find(sentence_pre[i])
+                if temp_id<0:
+                    if sentence_pre[i].find('"')>=0:
+                        temp_id = sentence_ori.find(sentence_pre[i].replace('"','" '))
+                    else:
+                        print('resotr index error:',sentence_pre[i])
+                new_sentence+=sentence_ori[0:temp_id]
+                new_sentence+=sentence_ori[temp_id:temp_id+len(sentence_pre[i])]
+                sentence_ori=sentence_ori[temp_id+len(sentence_pre[i]):]
+#    print('resotre:',restore_result)
+    return restore_result
+
+if __name__=='__main__':
+    path='//panfs/pan1/bionlp/lulab/luoling/HPO_project/bioTag/data/test/gsc/result/'
+    fin=open(path+'GSCplus_Nest_biobert.tsv','r',encoding='utf-8')
+    fout=open(path+'GSCplus_Nest_restore_biobert.tsv','w',encoding='utf-8')
+    all_context=fin.read().strip().split('\n\n\n\n')
+    fin.close()
+    file_num=0
+    for doc in all_context:
+        file_num+=1
+        print('file_num:',file_num)
+        doc_ele=doc.split('\n\n')
+        first_line = doc_ele[0].split('\n')
+        pmid=first_line[0]
+        ori_text=first_line[1]
+        pre_result='\n\n'.join(doc_ele[1:])
+#        print('pmid:',pmid)
+#        print('ori:',ori_text)
+#        print('pre:',pre_result)
+        final_result=restore_index_nest_fn(ori_text,pre_result)
+        fout.write(pmid+'\n'+ori_text+'\n')
+        for ele in final_result:
+            fout.write('\t'.join(ele)+'\t'+ori_text[int(ele[0]):int(ele[1])]+'\n')
+        fout.write('\n')
+    fout.close()

src/src_app-old.py

@@ -0,0 +1,268 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Mon Nov 21 16:21:25 2022
+
+@author: luol2
+"""
+
+import streamlit as st
+from src.nn_model import bioTag_CNN,bioTag_BERT
+from src.dic_ner import dic_ont
+from src.tagging_text import bioTag
+import os
+import json
+from pandas import DataFrame
+import nltk 
+nltk.download('punkt')
+nltk.download('averaged_perceptron_tagger')
+nltk.download('wordnet')
+
+st.set_page_config(
+    page_title="PhenoTagger_v1.2",
+    page_icon="🎈",
+    layout="wide",
+    menu_items={
+        'Get Help': 'https://www.ncbi.nlm.nih.gov/research/bionlp/',
+        'About': "PhenoTagger v1.2"
+    }
+)
+
+
+# def _max_width_():
+#     max_width_str = f"max-width: 2400px;"
+#     st.markdown(
+#         f"""
+#     <style>
+#     .reportview-container .main .block-container{{
+#         {max_width_str}
+#     }}
+#     </style>    
+#     """,
+#         unsafe_allow_html=True,
+#     )
+
+
+# _max_width_()
+
+# c30, c31, c32 = st.columns([2.5, 1, 3])
+
+# with c30:
+#     # st.image("logo.png", width=400)
+st.title("👨‍⚕️ PhenoTagger_v1.2 Demo")
+
+with st.expander("ℹ️ - About this app", expanded=True):
+
+    st.write(
+        """     
+-   This app is an easy-to-use interface built in Streamlit for [PhenoTagger](https://github.com/ncbi-nlp/PhenoTagger) library!
+-   PhenoTagger is a hybrid method that combines dictionary and deep learning-based methods to recognize Human Phenotype Ontology (HPO) concepts in unstructured biomedical text. Please refer to [our paper](https://doi.org/10.1093/bioinformatics/btab019) for more details.
+-   Contact: [NLM/NCBI BioNLP Research Group](https://www.ncbi.nlm.nih.gov/research/bionlp/)
+	    """
+    )
+
+    st.markdown("")
+
+st.markdown("")
+st.markdown("## 📌 Paste document ")
+with st.form(key="my_form"):
+
+
+    ce, c1, ce, c2, c3 = st.columns([0.07, 1, 0.07, 4, 0.07])
+    with c1:
+        ModelType = st.radio(
+            "Choose your Ontology",
+            ["HPO(Default)", "UBERON"],
+            #help="Bioformer is more precise, CNN is more efficient",
+        )
+
+        if ModelType == "HPO(Default)":
+            # kw_model = KeyBERT(model=roberta)
+
+            @st.cache(allow_output_mutation=True)
+            def load_model():
+                ontfiles={'dic_file':'./dict_hpo/noabb_lemma.dic',
+                          'word_id_file':'./dict_hpo/word_id_map.json',
+                          'id_word_file':'./dict_hpo/id_word_map.json'}
+        
+
+                vocabfiles={'labelfile':'./dict_hpo/lable.vocab',
+                            'checkpoint_path':'./models_v1.2/bioformer-cased-v1.0/',
+                            'lowercase':False}
+                modelfile='./models_v1.2/bioformer-HPO.h5'
+                   
+    
+                biotag_dic=dic_ont(ontfiles)    
+    
+                nn_model=bioTag_BERT(vocabfiles)
+                nn_model.load_model(modelfile)
+                return nn_model,biotag_dic
+
+            nn_model,biotag_dic = load_model()
+
+        else:
+            @st.cache(allow_output_mutation=True)
+            def load_model():
+                ontfiles={'dic_file':'./dict_uberon/noabb_lemma.dic',
+                          'word_id_file':'./dict_uberon/word_id_map.json',
+                          'id_word_file':'./dict_uberon/id_word_map.json'}
+
+                vocabfiles={'labelfile':'./dict_uberon/lable.vocab',
+                            'checkpoint_path':'./models_v1.2/bioformer-cased-v1.0/',
+                            'lowercase':False}
+                
+                modelfile='./models_v1.2/bioformer-UBERON.h5'
+        
+                biotag_dic=dic_ont(ontfiles)    
+            
+                nn_model=bioTag_CNN(vocabfiles)
+                nn_model.load_model(modelfile)
+            
+                return nn_model,biotag_dic
+
+            nn_model,biotag_dic = load_model()
+        
+        para_overlap = st.checkbox(
+            "Overlap concept",
+            value=False,
+            help="Tick this box to identify overlapping concepts",
+        )
+        para_abbr = st.checkbox(
+            "Abbreviaitons",
+            value=True,
+            help="Tick this box to identify abbreviations",
+        )        
+        
+        para_threshold = st.slider(
+            "Threshold",
+            min_value=0.5,
+            max_value=1.0,
+            value=0.95,
+            step=0.05,
+            help="Retrun the preditions which socre over the threshold.",
+        )
+      
+
+
+
+    with c2:
+        
+            
+        doc = st.text_area(
+              "Paste your text below",
+              value = 'The clinical features of Angelman syndrome (AS) comprise severe mental retardation, postnatal microcephaly, macrostomia and prognathia, absence of speech, ataxia, and a happy disposition. We report on seven patients who lack most of these features, but presented with obesity, muscular hypotonia and mild mental retardation. Based on the latter findings, the patients were initially suspected of having Prader-Willi syndrome. DNA methylation analysis of SNRPN and D15S63, however, revealed an AS pattern, ie the maternal band was faint or absent. Cytogenetic studies and microsatellite analysis demonstrated apparently normal chromosomes 15 of biparental inheritance. We conclude that these patients have an imprinting defect and a previously unrecognised form of AS. The mild phenotype may be explained by an incomplete imprinting defect or by cellular mosaicism.',
+              height=400,
+        )
+        
+
+        
+
+        # MAX_WORDS = 500
+        # import re
+        # res = len(re.findall(r"\w+", doc))
+        # if res > MAX_WORDS:
+        #     st.warning(
+        #         "⚠️ Your text contains "
+        #         + str(res)
+        #         + " words."
+        #         + " Only the first 500 words will be reviewed. Stay tuned as increased allowance is coming! 😊"
+        #     )
+
+        #     doc = doc[:MAX_WORDS]
+
+        submit_button = st.form_submit_button(label="✨ Submit!")
+
+
+if not submit_button:
+    st.stop()
+
+#st.write(para_overlap,para_abbr,para_threshold)
+para_set={
+          #model_type':para_model, # cnn or bioformer
+          'onlyLongest': not para_overlap, # False: return overlap concepts, True only longgest
+          'abbrRecog':para_abbr,# False: don't identify abbr, True: identify abbr
+          'ML_Threshold':para_threshold,# the Threshold of deep learning model
+          }
+st.markdown("")
+st.markdown("## 💡 Tagging results:")
+with st.spinner('Wait for tagging...'):
+    tag_result=bioTag(doc,biotag_dic,nn_model,onlyLongest=para_set['onlyLongest'], abbrRecog=para_set['abbrRecog'],Threshold=para_set['ML_Threshold'])
+
+st.markdown('<font style="color: rgb(128, 128, 128);">Move the mouse🖱️ over the entity to display the HPO id.</font>', unsafe_allow_html=True)
+# print('dic...........:',biotag_dic.keys())
+# st.write('parameters:', para_overlap,para_abbr,para_threshold)
+
+html_results=''
+text_results=doc+'\n'
+entity_end=0
+hpoid_count={}
+if len(tag_result)>=0:
+    for ele in tag_result:
+        entity_start=int(ele[0])
+        html_results+=doc[entity_end:entity_start]
+        entity_end=int(ele[1])
+        entity_id=ele[2]
+        entity_score=ele[3]
+        text_results+=ele[0]+'\t'+ele[1]+'\t'+doc[entity_start:entity_end]+'\t'+ele[2]+'\t'+format(float(ele[3]),'.2f')+'\n'
+        if entity_id not in hpoid_count.keys():
+            hpoid_count[entity_id]=1
+        else:
+            hpoid_count[entity_id]+=1
+        
+        html_results+='<font style="background-color: rgb(255, 204, 0)'+';" title="'+entity_id+'">'+doc[entity_start:entity_end]+'</font>'
+    html_results+=doc[entity_end:]
+                
+else:
+    html_results=doc
+    
+st.markdown('<table border="1"><tr><td>'+html_results+'</td></tr></table>', unsafe_allow_html=True)
+
+
+#table
+data_entity=[]
+for ele in hpoid_count.keys():
+    segs=ele.split(';')
+    term_name=''
+    for seg in segs:
+        term_name+=biotag_dic.id_word[seg][0]+';'
+    temp=[ele,term_name,hpoid_count[ele]] #hpoid, term name, count
+    data_entity.append(temp)
+
+
+st.markdown("")
+st.markdown("")
+# st.markdown("## Table output:")
+
+# cs, c1, c2, c3, cLast = st.columns([2, 1.5, 1.5, 1.5, 2])
+
+# with c1:
+#     CSVButton2 = download_button(keywords, "Data.csv", "📥 Download (.csv)")
+# with c2:
+#     CSVButton2 = download_button(keywords, "Data.txt", "📥 Download (.txt)")
+# with c3:
+#     CSVButton2 = download_button(keywords, "Data.json", "📥 Download (.json)")
+
+# st.header("")
+
+df = (
+    DataFrame(data_entity, columns=["Ontology_id", "Term name","Frequency"])
+    .sort_values(by="Frequency", ascending=False)
+    .reset_index(drop=True)
+)
+
+df.index += 1
+
+c1, c2, c3 = st.columns([1, 4, 1])
+
+# format_dictionary = {
+#     "Relevancy": "{:.1%}",
+# }
+
+# df = df.format(format_dictionary)
+
+with c2:
+    st.table(df)
+    
+c1, c2, c3 = st.columns([1, 1, 1])
+with c2:
+    st.download_button('Download annotations', text_results) 
+    

src/ssplit_tokenzier.py

@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 15:26:44 2020
+
+@author: luol2
+"""
+
+import nltk
+from nltk.stem import WordNetLemmatizer
+from nltk.corpus import wordnet
+from nltk.stem.porter import PorterStemmer
+lemmatizer = WordNetLemmatizer() 
+stemmer = PorterStemmer()
+import io
+    
+def get_wordnet_pos(treebank_tag):
+    if treebank_tag.startswith('J'):
+        return wordnet.ADJ
+    elif treebank_tag.startswith('V'):
+        return wordnet.VERB
+    elif treebank_tag.startswith('N'):
+        return wordnet.NOUN
+    elif treebank_tag.startswith('R') or treebank_tag=='IN':
+        return wordnet.ADV
+    else:
+        return wordnet.NOUN
+    
+def ssplit_token_pos_lemma(in_text):
+    
+    fout=io.StringIO()
+
+    line=in_text.strip()
+    line=line.replace('-',' - ').replace('/',' / ')
+    sentences = nltk.sent_tokenize(line)
+    sentences = [nltk.word_tokenize(sent) for sent in sentences] 
+#    print(sentences)
+    for sent in sentences:
+        token_pos = nltk.pos_tag(sent)
+        for token in token_pos:
+            lemma = lemmatizer.lemmatize(token[0].lower(), get_wordnet_pos(token[1]))
+            stem = stemmer.stem(token[0].lower())
+            fout.write(token[0]+'\t'+lemma+'\t'+stem+'\t'+token[1]+'\n')
+        fout.write('\n')
+           
+    return fout.getvalue()

src/tagging_text.py

@@ -0,0 +1,98 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Fri Jun 12 11:33:22 2020
+
+@author: luol2
+"""
+import argparse
+from src.ssplit_tokenzier import ssplit_token_pos_lemma
+from src.ml_ner import ml_tagging,ml_tagging_allngram
+from src.combine_result import combine_ml_dict
+from src.restore_index import restore_index_nest_fn
+from src.dic_ner import dic_ont
+from src.post_processing import combine_overlap
+from src.abbre_resolution import postprocess_abbr
+import os
+import time
+import json
+
+#hybrid method
+def bioTag(text,biotag_dic,ml_model,onlyLongest=False, abbrRecog=False, Threshold=0.95):
+
+#    startTime=time.time()
+    ssplit_token=ssplit_token_pos_lemma(text)
+    #print(ssplit_token) 
+#    print('ssplit token:',time.time()-startTime)
+    
+#    startTime=time.time()
+    dict_tsv=biotag_dic.matching(ssplit_token)
+#    print('dict tsv:\n',dict_tsv)
+#    print('dict ner:',time.time()-startTime)
+  
+#    startTime=time.time()
+    ml_tsv=ml_tagging(ssplit_token,ml_model,Threshold)
+    #print('ml_tsv:\n',ml_tsv)
+#    print('ml ner:',time.time()-startTime)
+    
+#    startTime=time.time()
+    combine_tsv=combine_ml_dict(dict_tsv,ml_tsv)
+    #combine_tsv=combine_ml_dict_fn(ml_tsv,dict_tsv)
+    #print('combine:\n',combine_tsv)
+    
+    final_result=  restore_index_nest_fn(text,combine_tsv)
+#    print('final ner:',time.time()-startTime)
+    if onlyLongest==True:
+        final_result=combine_overlap(final_result)
+    if abbrRecog==True:
+        final_result=postprocess_abbr(final_result,text)
+#    print('final result:') 
+#    print(final_result)
+    
+    return final_result
+
+# only machine learning-based method
+def bioTag_ml(text,ml_model,onlyLongest=False,abbrRecog=False, Threshold=0.95):
+
+#    startTime=time.time()
+    ssplit_token=ssplit_token_pos_lemma(text)
+#    print(ssplit_token) 
+#    print('ssplit token:',time.time()-startTime)
+    
+#    startTime=time.time()
+    ml_tsv=ml_tagging_allngram(ssplit_token,ml_model,Threshold)
+#    print('ml_tsv:\n',ml_tsv)
+#    print('ml ner:',time.time()-startTime)
+       
+    final_result=  restore_index_nest_fn(text,ml_tsv)
+#    print('final ner:',time.time()-startTime)
+    if onlyLongest==True:
+        final_result=combine_overlap(final_result)
+    
+    if abbrRecog==True:
+        final_result=postprocess_abbr(final_result,text)
+    
+    return final_result
+
+# only dict method
+def bioTag_dic(text,biotag_dic,onlyLongest=False, abbrRecog=False):
+
+#    startTime=time.time()
+    ssplit_token=ssplit_token_pos_lemma(text)
+#    print(ssplit_token) 
+#    print('ssplit token:',time.time()-startTime)
+    
+#    startTime=time.time()
+    dict_tsv=biotag_dic.matching(ssplit_token)
+#    print('dict tsv:\n',dict_tsv)
+#    print('dict ner:',time.time()-startTime)
+    
+    final_result=  restore_index_nest_fn(text,dict_tsv)
+#    print('final ner:',time.time()-startTime)
+    if onlyLongest==True:
+        final_result=combine_overlap(final_result)
+    
+    if abbrRecog==True:
+        final_result=postprocess_abbr(final_result,text)
+    
+    return final_result
+