import time import streamlit as st import torch import string bert_tokenizer = None bert_model = None from transformers import BertTokenizer, BertForMaskedLM st.set_page_config(page_title='Qualitative pretrained model eveluation', page_icon=None, layout='centered', initial_sidebar_state='auto') @st.cache() def load_bert_model(model_name): try: bert_tokenizer = BertTokenizer.from_pretrained(model_name,do_lower_case =False) bert_model = BertForMaskedLM.from_pretrained(model_name).eval() return bert_tokenizer,bert_model except Exception as e: pass def decode(tokenizer, pred_idx, top_clean): ignore_tokens = string.punctuation tokens = [] for w in pred_idx: token = ''.join(tokenizer.decode(w).split()) if token not in ignore_tokens and len(token) > 1 and not token.startswith('.') and not token.startswith('['): #tokens.append(token.replace('##', '')) tokens.append(token) return '\n'.join(tokens[:top_clean]) def encode(tokenizer, text_sentence, add_special_tokens=True): text_sentence = text_sentence.replace('', tokenizer.mask_token) # if is the last token, append a "." so that models dont predict punctuation. #if tokenizer.mask_token == text_sentence.split()[-1]: # text_sentence += ' .' tokenized_text = bert_tokenizer.tokenize(text_sentence) input_ids = torch.tensor([tokenizer.encode(text_sentence, add_special_tokens=add_special_tokens)]) if (tokenizer.mask_token in text_sentence.split()): mask_idx = torch.where(input_ids == tokenizer.mask_token_id)[1].tolist()[0] else: mask_idx = 0 return input_ids, mask_idx,tokenized_text def get_all_predictions(text_sentence, model_name,top_clean=5): # ========================= BERT ================================= input_ids, mask_idx,tokenized_text = encode(bert_tokenizer, text_sentence) with torch.no_grad(): predict = bert_model(input_ids)[0] bert = decode(bert_tokenizer, predict[0, mask_idx, :].topk(top_k*5).indices.tolist(), top_clean) cls = decode(bert_tokenizer, predict[0, 0, :].topk(top_k*5).indices.tolist(), top_clean) if ("[MASK]" in text_sentence or "" in text_sentence): return {'Input sentence':text_sentence,'Tokenized text': tokenized_text,'Model':model_name,'Masked position': bert,'[CLS]':cls} else: return {'Input sentence':text_sentence,'Tokenized text': tokenized_text,'Model':model_name,'[CLS]':cls} def get_bert_prediction(input_text,top_k,model_name): try: #input_text += ' ' res = get_all_predictions(input_text,model_name, top_clean=int(top_k)) return res except Exception as error: pass def run_test(sent,top_k,model_name): start = None global bert_tokenizer global bert_model global display_container if (bert_tokenizer is None): bert_tokenizer, bert_model = load_bert_model(model_name) with st.spinner("Computing"): start = time.time() try: res = get_bert_prediction(sent,top_k,model_name) st.caption("Results in JSON") st.json(res) display_container.write("results") except Exception as e: st.error("Some error occurred during prediction" + str(e)) st.stop() if start is not None: st.text(f"prediction took {time.time() - start:.2f}s") def on_text_change(): global top_k,model_name text = st.session_state.my_text run_test(text,top_k,model_name) def on_option_change(): global top_k,model_name text = st.session_state.my_choice run_test(text,top_k,model_name) def init_selectbox(): option = st.selectbox( 'Choose any of these sentences or type any text below', ('', "[MASK] who lives in New York and works for XCorp suffers from Parkinson's", "Lou Gehrig who lives in [MASK] and works for XCorp suffers from Parkinson's","Lou Gehrig who lives in New York and works for [MASK] suffers from Parkinson's","Lou Gehrig who lives in New York and works for XCorp suffers from [MASK]","[MASK] who lives in New York and works for XCorp suffers from Lou Gehrig's", "Parkinson who lives in [MASK] and works for XCorp suffers from Lou Gehrig's","Parkinson who lives in New York and works for [MASK] suffers from Lou Gehrig's","Parkinson who lives in New York and works for XCorp suffers from [MASK]","Lou Gehrig","Parkinson","Lou Gehrigh's is a [MASK]","Parkinson is a [MASK]","New York is a [MASK]","New York","XCorp","XCorp is a [MASK]","acute lymphoblastic leukemia","acute lymphoblastic leukemia is a [MASK]"),on_change=on_option_change,key='my_choice') return option st.markdown("

Qualitative evaluation of any pretrained BERT model

", unsafe_allow_html=True) st.markdown(""" Pretrained BERT models can be used as is, with no fine tuning to perform tasks like NER ideally if both fill-mask and CLS predictions are good, or minimally if fill-mask predictions are adequate """, unsafe_allow_html=True) #st.write("https://ajitrajasekharan.github.io/2021/01/02/my-first-post.html") st.write("This app can be used to examine both model prediction for a masked position as well as the neighborhood of CLS vector") st.write(" - To examine model prediction for a position, enter the token [MASK] or ") st.write(" - To examine just the [CLS] vector, enter a word/phrase or sentence. Example: eGFR or EGFR or non small cell lung cancer") top_k = st.sidebar.slider("Select how many predictions do you need", 1 , 50, 20) #some times it is possible to have less words print(top_k) #if st.button("Submit"): # with st.spinner("Computing"): try: model_name = st.sidebar.selectbox(label='Select Model to Apply', options=['ajitrajasekharan/biomedical', 'bert-base-cased','bert-large-cased','microsoft/BiomedNLP-PubMedBERT-base-uncased-abstract-fulltext','allenai/scibert_scivocab_cased','dmis-lab/biobert-v1.1'], index=0, key = "model_name") option = init_selectbox() input_text = st.text_input("Enter text below", "",on_change=on_text_change,key='my_text') custom_model_name = st.text_input("Model not listed on left? Type the model name (fill-mask BERT models only)", "") if (len(custom_model_name) > 0): model_name = custom_model_name st.info("Custom model selected: " + model_name) bert_tokenizer, bert_model = load_bert_model(model_name) #if len(input_text) > 0: # run_test(input_text,top_k,model_name) #else: # if len(option) > 0: # run_test(option,top_k,model_name) display_container = st.container() display_container.write("ajits test") if (bert_tokenizer is None): bert_tokenizer, bert_model = load_bert_model(model_name) except Exception as e: st.error("Some error occurred during loading" + str(e)) st.stop() st.write("---")