File size: 3,274 Bytes
4a6159c cd8693c 4a6159c |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 |
from tkinter import Text
from haystack.nodes import TransformersDocumentClassifier
from typing import List, Tuple
import configparser
import streamlit as st
from pandas import DataFrame, Series
import logging
from udfPreprocess.preprocessing import processingpipeline
config = configparser.ConfigParser()
config.read_file(open('udfPreprocess/paramconfig.cfg'))
@st.cache(allow_output_mutation=True)
def load_sdgClassifier():
"""
loads the document classifier using haystack, where the name/path of model
in HF-hub as string is used to fetch the model object.
1. https://docs.haystack.deepset.ai/reference/document-classifier-api
2. https://docs.haystack.deepset.ai/docs/document_classifier
Return: document classifier model
"""
logging.info("Loading classifier")
doc_classifier_model = config.get('sdg','MODEL')
doc_classifier = TransformersDocumentClassifier(
model_name_or_path=doc_classifier_model,
task="text-classification")
return doc_classifier
def sdg_classification(paraList:List[Text])->Tuple[DataFrame,Series]:
"""
Text-Classification on the list of texts provided. Classifier provides the
most appropriate label for each text. these labels are in terms of if text
belongs to which particular Sustainable Devleopment Goal (SDG).
Params
---------
paraList: List of paragrpahs/text. The output of Preprocessing Pipeline
contains this list of paragraphs in different format, the simple List format
is being used here.
Returns
----------
df: Dataframe with two columns['SDG:int', 'text']
x: Series object with the unique SDG covered in the document uploaded and
the number of times it is covered/discussed/count_of_paragraphs.
"""
logging.info("running SDG classifiication")
threshold = float(config.get('sdg','THRESHOLD'))
classifier = load_sdgClassifier()
labels = classifier(paraList)
labels_= [(l['label'],l['score']) for l in labels]
df = DataFrame(labels_, columns=["SDG", "Relevancy"])
df['text'] = paraList
df = df.sort_values(by="Relevancy", ascending=False).reset_index(drop=True)
df.index += 1
df =df[df['Relevancy']>threshold]
x = df['SDG'].value_counts()
# df = df.copy()
df= df.drop(['Relevancy'], axis = 1)
return df, x
def runSDGPreprocessingPipeline()->List[Text]:
"""
creates the pipeline and runs the preprocessing pipeline,
the params for pipeline are fetched from paramconfig
"""
file_path = st.session_state['filepath']
file_name = st.session_state['filename']
sdg_processing_pipeline = processingpipeline()
split_by = config.get('sdg','SPLIT_BY')
split_length = int(config.get('sdg','SPLIT_LENGTH'))
output_sdg_pre = sdg_processing_pipeline.run(file_paths = file_path,
params= {"FileConverter": {"file_path": file_path, \
"file_name": file_name},
"UdfPreProcessor": {"removePunc": False, \
"split_by": split_by, \
"split_length":split_length}})
return output_sdg_pre['paraList']
|