Spaces:
Sleeping
Sleeping
import re | |
import spacy_streamlit | |
from spacy_streamlit import visualize_parser | |
from collections import Counter | |
import spacy | |
import streamlit as st | |
# try: | |
# from .scripts.custom_functions import build_mean_max_reducer1, build_mean_max_reducer2, build_mean_max_reducer3 | |
# except ImportError: | |
# from pipeline.custom_functions import build_mean_max_reducer1, build_mean_max_reducer2, build_mean_max_reducer3 | |
from spacy.tokens import Doc | |
from spacy.cli._util import import_code | |
from utils.visualize import visualize_spans | |
from utils.utility import preprocess, delete_overlapping_span, cleanup_justify | |
from resources.text_list import TEXT_LIST | |
from resources.text_list_BAWE import TEXT_LIST_BAWE | |
from resources.template_list import TPL_SPAN, TPL_SPAN_SLICE, TPL_SPAN_START | |
from resources.colors import COLORS_1 | |
import_code("pipeline/custom_functions.py") | |
st.set_page_config(page_title='Engagement model comparaer', layout="wide") | |
# spacy.prefer_gpu() | |
MODEL_LIST =['en_engagement_LSTM', 'en_engagement_LSTM'] | |
# MODEL_LIST = [ | |
# 'en_engagement_three_RoBERTa_base_LSTM384-0.9.2/en_engagement_three_RoBERTa_base_LSTM384/en_engagement_three_RoBERTa_base_LSTM384-0.9.2', | |
# 'en_engagement_three_RoBERTa_acad3_db-0.9.2/en_engagement_three_RoBERTa_acad3_db/en_engagement_three_RoBERTa_acad3_db-0.9.2', | |
# 'silver-sweep-34/model-best', | |
# 'expert-sweep-4/model-best', | |
# 'confused-sweep-6/model-best', | |
# 'warm-sweep-20/model-best', | |
# "en_engagement_three_RoBERTa_base-1.10.0/en_engagement_three_RoBERTa_base/en_engagement_three_RoBERTa_base-1.10.0", | |
# "en_engagement_three_RoBERTa_acad_db-1.10.0/en_engagement_three_RoBERTa_acad_db/en_engagement_three_RoBERTa_acad_db-1.10.0", | |
# "en_engagement_para_RoBERTa_acad_db3-0.9.0/en_engagement_para_RoBERTa_acad_db3/en_engagement_para_RoBERTa_acad_db3-0.9.0", | |
# "en_engagement_para_RoBERTa_acad_LSTM2-0.9.0/en_engagement_para_RoBERTa_acad_LSTM2/en_engagement_para_RoBERTa_acad_LSTM2-0.9.0", | |
# "en_engagement_three_RoBERTa_acad_db3-0.9.1/en_engagement_three_RoBERTa_acad_db3/en_engagement_three_RoBERTa_acad_db3-0.9.1", | |
# "en_engagement_three_RoBERTa_acad_LSTM2-0.9.1/en_engagement_three_RoBERTa_acad_LSTM2/en_engagement_three_RoBERTa_acad_LSTM2-0.9.1", | |
# "en_engagement_three_RoBERTa_acad_db3-0.9.2/en_engagement_three_RoBERTa_acad_db3/en_engagement_three_RoBERTa_acad_db3-0.9.2", | |
# 'en_engagement_spl_RoBERTa_acad_db-0.7.4/en_engagement_spl_RoBERTa_acad_db/en_engagement_spl_RoBERTa_acad_db-0.7.4', | |
# 'en_engagement_spl_RoBERTa_acad_db3-0.9.0/en_engagement_spl_RoBERTa_acad_db3/en_engagement_spl_RoBERTa_acad_db3-0.9.0', | |
# 'en_engagement_spl_RoBERTa_acad_LSTM-0.7.2/en_engagement_spl_RoBERTa_acad_LSTM/en_engagement_spl_RoBERTa_acad_LSTM-0.7.2', | |
# 'en_engagement_spl_RoBERTa_acad_512', | |
# 'en_engagement_spl_RoBERTa_acad', | |
# 'en_engagement_spl_RoBERTa_exp-0.6.5/en_engagement_spl_RoBERTa_exp/en_engagement_spl_RoBERTa_exp-0.6.5', | |
# # 'en_engagement_spl_RoBERTa_acad-0.3.4.1221/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.3.4.1221', | |
# # 'en_engagement_spl_RoBERTa_acad-0.2.2.1228/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.2.1228', | |
# # 'en_engagement_spl_RoBERTa_acad-0.2.1.1228/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.1.1228', | |
# # 'en_engagement_spl_RoBERTa_acad-0.2.2.1220/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.2.1220', | |
# # 'en_engagement_spl_RoBERTa2-0.2.2.1210/en_engagement_spl_RoBERTa2/en_engagement_spl_RoBERTa2-0.2.2.1210', | |
# # 'en_engagement_spl_RoBERTa-0.2.2.1210/en_engagement_spl_RoBERTa/en_engagement_spl_RoBERTa-0.2.2.1210', | |
# # 'en_engagement_spl_RoBERTa_acad_max1_do02', | |
# # 'en_engagement_spl_RoBERTa2-0.2.2.1210/en_engagement_spl_RoBERTa2/en_engagement_spl_RoBERTa2-0.2.2.1210', | |
# # 'en_engagement_spl_RoBERTa_acad-0.2.3.1210/en_engagement_spl_RoBERTa_acad/en_engagement_spl_RoBERTa_acad-0.2.3.1210', | |
# # 'en_engagement_spl_RoBERTa_acad_max1_do02', | |
# # 'en_engagement_spl_RoBERTa_sqbatch_RAdam-20221202_0.1.5/en_engagement_spl_RoBERTa_sqbatch_RAdam/en_engagement_spl_RoBERTa_sqbatch_RAdam-20221202_0.1.5', | |
# # 'en_engagement_spl_RoBERTa_context_flz-20221130_0.1.4/en_engagement_spl_RoBERTa_context_flz/en_engagement_spl_RoBERTa_context_flz-20221130_0.1.4', | |
# # 'en_engagement_spl_RoBERTa_cx_max1_do2-20221202_0.1.5/en_engagement_spl_RoBERTa_cx_max1_do2/en_engagement_spl_RoBERTa_cx_max1_do2-20221202_0.1.5', | |
# # 'en_engagement_spl_RoBERTa_context_flz-20221125_0.1.4/en_engagement_spl_RoBERTa_context_flz/en_engagement_spl_RoBERTa_context_flz-20221125_0.1.4', | |
# # 'en_engagement_RoBERTa_context_flz-20221125_0.1.4/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221125_0.1.4', | |
# # 'en_engagement_RoBERTa_context_flz-20221117_0.1.3/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221117_0.1.3', | |
# # 'en_engagement_spl_RoBERTa_acad_context_flz-20221117_0.1.3/en_engagement_spl_RoBERTa_acad_context_flz/en_engagement_spl_RoBERTa_acad_context_flz-20221117_0.1.3', | |
# # 'en_engagement_RoBERTa_context_flz-Batch2_0.1.1/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-Batch2_0.1.1', | |
# # 'en_engagement_RoBERTa_context_flz-20221113_0.1.3/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221113_0.1.3', | |
# # 'en_engagement_RoBERTa_context_flz-20221113_0.1.1/en_engagement_RoBERTa_context_flz/en_engagement_RoBERTa_context_flz-20221113_0.1.1', | |
# # 'en_engagement_RoBERTa-0.0.2/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.2', | |
# # 'en_engagement_RoBERTa_combined-Batch2Eng_0.2/en_engagement_RoBERTa_combined/en_engagement_RoBERTa_combined-Batch2Eng_0.2', | |
# # 'en_engagement_RoBERTa_acad-0.2.1/en_engagement_RoBERTa_acad/en_engagement_RoBERTa_acad-0.2.1', | |
# # # 'en_engagement_BERT-0.0.2/en_engagement_BERT/en_engagement_BERT-0.0.2', | |
# # # 'en_engagement_BERT_acad-0.0.2/en_engagement_BERT_acad/en_engagement_BERT_acad-0.0.2', | |
# # # 'en_engagement_RoBERTa_acad-0.0.2/en_engagement_RoBERTa_acad/en_engagement_RoBERTa_acad-0.0.2', | |
# # 'en_engagement_RoBERTa-0.0.1/en_engagement_RoBERTa/en_engagement_RoBERTa-0.0.1', | |
# # # ' en_engagement_RoBERTa_sent-0.0.1_null/en_engagement_RoBERTa_sent/en_engagement_RoBERTa_sent-0.0.1_null', | |
# # # 'en_engagement_RoBERTa_combined-0.0.1/en_engagement_RoBERTa_combined/en_engagement_RoBERTa_combined-0.0.1', | |
# # 'en_engagement_RoBERTa-ME_AtoE/en_engagement_RoBERTa/en_engagement_RoBERTa-ME_AtoE', | |
# # 'en_engagement_RoBERTa-AtoI_0.0.3/en_engagement_RoBERTa/en_engagement_RoBERTa-AtoI_0.0.3', | |
# # 'en_engagement_RoBERTa-AtoI_0.0.3/en_engagement_RoBERTa/en_engagement_RoBERTa-AtoI_0.0.2' | |
# ] | |
multicol = st.checkbox("Compare two models", value=False, key=None, help=None) | |
model1 = st.selectbox('Select model option 1', MODEL_LIST, index=0) | |
model2 = st.selectbox('Select model option 2', MODEL_LIST, index=1) | |
if '/' in model1: | |
model1 = "packages/" + model1 | |
if '/' in model2: | |
model2 = "packages/" + model2 | |
def load_model(spacy_model): | |
# source = spacy.blank("en") | |
nlp = spacy.load(spacy_model) #, vocab=nlp_to_copy.vocab | |
nlp.add_pipe('sentencizer') | |
return (nlp) | |
# source = spacy.blank("en") | |
nlp = load_model(model1) | |
if multicol: | |
nlp2 = load_model(model2) | |
text = st.selectbox('select sent to debug', TEXT_LIST_BAWE) | |
input_text = st.text_area("", height=200) | |
# Dependency parsing | |
st.header("Text", "text") | |
if len(input_text.split(" ")) > 1: | |
doc = nlp(preprocess(input_text)) | |
if multicol: | |
doc2 = nlp2(preprocess(input_text)) | |
# st.markdown("> " + input_text) | |
else: | |
doc = nlp(preprocess(text)) | |
if multicol: | |
doc2 = nlp2(preprocess(text)) | |
# st.markdown("> " + text) | |
clearjustify = st.checkbox( | |
"Clear problematic JUSTIFYING spans", value=True, key=None, help=None) | |
delete_overlaps = st.checkbox( | |
"Delete overlaps", value=True, key=None, help=None) | |
# combine = st.checkbox( | |
# "Combine", value=False, key=None, help=None) | |
# import copy | |
# def combine_spangroups(doc1, doc2): | |
# # new_doc = Doc.from_docs([doc1, doc2], ensure_whitespace=True) | |
# new_doc = copy.deepcopy(doc1) | |
# # type() | |
# new_doc.spans['sc'].extend(doc2.spans['sc']) | |
# return new_doc | |
# if combine: | |
# new_doc = combine_spangroups(doc, doc2) | |
# visualize_spans(new_doc, | |
# spans_key="sc", | |
# title='Combined spans:', | |
# displacy_options={ | |
# 'template': { | |
# "span": TPL_SPAN, | |
# 'slice': TPL_SPAN_SLICE, | |
# 'start': TPL_SPAN_START, | |
# }, | |
# "colors": COLORS_1, | |
# }, | |
# simple=False) | |
if clearjustify: | |
cleanup_justify(doc, doc.spans['sc']) | |
if delete_overlaps: | |
delete_overlapping_span(doc.spans['sc']) | |
if multicol: | |
delete_overlapping_span(doc2.spans['sc']) | |
if not multicol: | |
visualize_spans(doc, | |
spans_key="sc", | |
title='Engagement Span Anotations 1', | |
displacy_options={ | |
'template': { | |
"span": TPL_SPAN, | |
'slice': TPL_SPAN_SLICE, | |
'start': TPL_SPAN_START, | |
}, | |
"colors": COLORS_1, | |
}, | |
simple=False) | |
else: | |
col1, col2 = st.columns(2) | |
with col1: | |
visualize_spans(doc, | |
spans_key="sc", | |
title='Engagement Span Anotations 1', | |
displacy_options={ | |
'template': { | |
"span": TPL_SPAN, | |
'slice': TPL_SPAN_SLICE, | |
'start': TPL_SPAN_START, | |
}, | |
"colors": COLORS_1, | |
}, | |
simple=False) | |
with col2: | |
visualize_spans(doc2, | |
spans_key="sc", | |
title='Engagement Span Anotations 2', | |
displacy_options={ | |
'template': { | |
"span": TPL_SPAN, | |
'slice': TPL_SPAN_SLICE, | |
'start': TPL_SPAN_START, | |
}, | |
"colors": COLORS_1, | |
}, | |
simple=False) | |
dep_options = {"fine_grained": True, "distance": 120} | |
visualize_parser(doc, displacy_options=dep_options) |