Added CSV and Dataframe inputs (evaluation still WIP)

app.py

@@ -16,14 +16,46 @@ nltk.download("punkt")
 
 def run(the_method, text, compression_ratio, use_golden=False, golden=None):
     if the_method[0:4] == "Sumy":
-        return run_sumy(the_method, _clean_text(text), compression_ratio, golden), run_eval(use_golden, _clean_text(text), run_sumy(the_method, _clean_text(text), compression_ratio, golden), golden)
+        return run_sumy(the_method, _clean_text(text), compression_ratio), run_eval(use_golden, _clean_text(text), run_sumy(the_method, _clean_text(text), compression_ratio), golden)
     elif the_method[0:13] == "Transformers-":
-        return run_transformers(the_method, _clean_text(text), compression_ratio, golden), run_eval(use_golden, _clean_text(text), run_transformers(the_method, _clean_text(text), compression_ratio, golden), golden)
+        return run_transformers(the_method, _clean_text(text), compression_ratio), run_eval(use_golden, _clean_text(text), run_transformers(the_method, _clean_text(text), compression_ratio), golden)
 
+def run_csv(the_method, csv_input, text_column, compression_ratio=1 / 8, use_golden=False):
+    df_original = pd.read_csv(csv_input.name)
+    text_series = df_original[text_column]
+    text_series = text_series.apply(lambda x: _clean_text(x))
 
-def run_csv(the_method, csv_input, compression_ratio=1 / 8, use_golden=False):
-    pass
+    if the_method[0:4] == "Sumy":
+        result = run_sumy_df(the_method, text_series, compression_ratio)
+        the_method_dir = the_method[4:]
+        #run_eval(use_golden, df, run_sumy(the_method, df, compression_ratio))
+    elif the_method[0:13] == "Transformers-":
+        the_method_dir = re.sub(r"[\/]","-",the_method[13:])
+        result = run_transformers_df(the_method, text_series, compression_ratio)
+        #run_eval(use_golden, df, run_sumy(the_method, df, compression_ratio))
+
+    column_name = "summary_"+the_method_dir
+    df_original[column_name] = result["summary"]
+    df_original.to_csv(the_method_dir+"_results.csv", index=False)
+    return str(the_method_dir+"_results.csv")
+
+
+def run_df(the_method, df, compression_ratio=1 / 8, use_golden=False):
+    
+    text_series = df.iloc[:,0].apply(lambda x: _clean_text(x))
+    print(text_series)
 
+    if the_method[0:4] == "Sumy":
+        result = run_sumy_df(the_method, text_series, compression_ratio)
+        the_method_dir = the_method[4:]
+        #run_eval(use_golden, df, run_sumy(the_method, df, compression_ratio))
+    elif the_method[0:13] == "Transformers-":
+        the_method_dir = re.sub(r"[\/]","-",the_method[13:])
+        result = run_transformers_df(the_method, text_series, compression_ratio)
+        #run_eval(use_golden, df, run_sumy(the_method, df, compression_ratio))
+
+    result.to_csv(the_method_dir+"_results.csv", index=False)
+    return str(the_method_dir+"_results.csv")
 
 def _clean_text(content):
     if isinstance(content, str):
@@ -55,7 +87,7 @@ def _clean_text(content):
     return content
 
 
-def run_sumy(method, text, compression_ratio, golden):
+def run_sumy(method, text, compression_ratio):
     from sumy.summarizers.random import RandomSummarizer
     from sumy.summarizers.luhn import LuhnSummarizer
     from sumy.summarizers.lsa import LsaSummarizer
@@ -66,40 +98,14 @@ def run_sumy(method, text, compression_ratio, golden):
     from sumy.summarizers.reduction import ReductionSummarizer
     from sumy.summarizers.edmundson import EdmundsonSummarizer
 
-    def word_frequency(golden, text, n=20):
-
-        sum_tokens = [t.lower() for t in word_tokenize(golden) if t not in stopwords.words("english") and t.isalpha()]
-        print(sum_tokens)
-        sum_word_freq_descending = pd.DataFrame(Counter(sum_tokens).items(), columns=["word", "frequency sum"]).sort_values(by="frequency sum", ascending=False)
-
-        texts_tokens = [t.lower() for t in word_tokenize(text) if t not in stopwords.words("english") and t.isalpha()]
-        print(texts_tokens)
-        texts_word_freq_descending = pd.DataFrame(Counter(texts_tokens).items(), columns=["word", "frequency text"]).sort_values(by="frequency text", ascending=False)
-
-        stigma_words = pd.merge(sum_word_freq_descending, texts_word_freq_descending, on="word")
-        stigma_words["frequency"] = stigma_words["frequency text"] / stigma_words["frequency sum"]
-        stigma_words = stigma_words.sort_values(by="frequency", ascending=False)
-
-        stigma_words = stigma_words["word"].tolist()[:n]
-        bonus_words = sum_word_freq_descending["word"].tolist()[:n]
-        return bonus_words, stigma_words
-
     the_method = method.replace("Sumy", "")
     summarizer = locals()[the_method + "Summarizer"]()
     sentence_count = int(len(sent_tokenize(text)) * compression_ratio / 100)
+    if sentence_count < 1:
+        sentence_count = 1
     parser = PlaintextParser.from_string(text, Tokenizer("english"))
 
-    if the_method != "Edmundson":
-        summary = summarizer(parser.document, sentence_count)
-    else:
-        bonus_words, stigma_words = word_frequency(golden, text, 10)
-        summarizer = EdmundsonSummarizer(cue_weight=1, key_weight=1, title_weight=0, location_weight=0)
-        summarizer.bonus_words = bonus_words
-        summarizer.stigma_words = stigma_words
-        summarizer.null_words = stopwords.words("english")
-        print(bonus_words)
-        print(stigma_words)
-        summary = summarizer(parser.document, sentence_count)
+    summary = summarizer(parser.document, sentence_count)
 
     text_summary = ""
     for s in summary:
@@ -107,7 +113,7 @@ def run_sumy(method, text, compression_ratio, golden):
     return text_summary
 
 
-def run_transformers(method, text, compression_ratio, golden):
+def run_transformers(method, text, compression_ratio):
 
     the_method = method.replace("Transformers-", "")
     summarizer = pipeline("summarization", model=the_method)
@@ -121,6 +127,68 @@ def run_transformers(method, text, compression_ratio, golden):
     return summary
 
 
+def run_sumy_df(method, texts_series, compression_ratio):
+    
+    from sumy.summarizers.random import RandomSummarizer
+    from sumy.summarizers.luhn import LuhnSummarizer
+    from sumy.summarizers.lsa import LsaSummarizer
+    from sumy.summarizers.lex_rank import LexRankSummarizer
+    from sumy.summarizers.text_rank import TextRankSummarizer
+    from sumy.summarizers.sum_basic import SumBasicSummarizer
+    from sumy.summarizers.kl import KLSummarizer
+    from sumy.summarizers.reduction import ReductionSummarizer
+    from sumy.summarizers.edmundson import EdmundsonSummarizer
+    from sumy.parsers.plaintext import PlaintextParser
+    from sumy.nlp.tokenizers import Tokenizer  # For Strings
+    from sumy.parsers.html import HtmlParser
+    from sumy.utils import get_stop_words
+    from nltk.tokenize import word_tokenize
+    from nltk.corpus import stopwords
+    from nltk.stem import WordNetLemmatizer
+    from collections import Counter
+
+    the_method = method.replace("Sumy", "")
+    the_summarizer = locals()[the_method + "Summarizer"]()
+
+    summarizer_output_list = []
+    for text in texts_series:
+        parser = PlaintextParser.from_string(text, Tokenizer("english"))
+        sentence_count = int(len(sent_tokenize(text)) * compression_ratio / 100)
+        if sentence_count < 1:
+            sentence_count = 1
+        summarizer_output_list.append(the_summarizer(parser.document, sentence_count))
+
+    candidate_summaries = []
+    for summarizer_output in summarizer_output_list:
+        text_summary = ""
+        for sentence in summarizer_output:
+            text_summary += str(sentence) + " "
+
+        candidate_summaries.append(text_summary)
+
+    results = pd.DataFrame({"text": texts_series, "summary": candidate_summaries})
+    return results
+
+def run_transformers_df(method, texts_series, compression_ratio):
+    from transformers import pipeline
+    from nltk.tokenize import word_tokenize
+
+    the_method = method.replace("Transformers-", "")
+    summarizer = pipeline("summarization", model=the_method)
+
+    aux_summaries_list = []
+    for text in texts_series:
+        length = 3000
+        while len(word_tokenize(text[0:length])) > 450:
+            length -= 100
+            token_count = len(word_tokenize(text[0:length])) * compression_ratio / 100
+        aux_summaries_list.append(summarizer(text[0:length], min_length=int(token_count - 5), max_length=int(token_count + 5)))
+
+    candidate_summaries = [x[0]["summary_text"] for x in aux_summaries_list]
+
+    results = pd.DataFrame({"text": texts_series, "summary": candidate_summaries})
+    return results
+
 def run_eval(use_golden, text, summary, golden):
     if use_golden:
         rouge = run_rouge_eval(summary, golden)
@@ -289,7 +357,7 @@ if __name__ == "__main__":
                     with gr.Column(scale=1, min_width=300):
                         compression_ratio = gr.Slider(
                             label="Taxa de Compressão (% do tamanho original)",
-                            value=30,
+                            value=10,
                             minimum=1,
                             maximum=100,
                         )
@@ -312,12 +380,14 @@ if __name__ == "__main__":
                 with gr.Tab("CSV"):
                     with gr.Column(scale=1, min_width=300):
                         gr.Checkbox(
-                            label="Insira abaixo um arquivo CSV com uma coluna de textos a serem sumarizados. Caso opte por avaliar usando golden summaries, estes deverão estar presentes em uma segunda coluna.",
+                            label="Insira abaixo um arquivo CSV com uma coluna de textos a serem sumarizados. Caso opte por avaliar usando golden summaries, estes deverão estar presentes em uma coluna entitulada 'golden'.",
                             value=False,
                             interactive=False,
                         )
                         with gr.Row():
-                            csv_input = gr.File(label="Arquivo .csv de textos")
+                            with gr.Column(scale=1, min_width=300):
+                                text_column = gr.Textbox(label="Título da coluna a ser sumarizada", placeholder="text")
+                                csv_input = gr.File(label="Arquivo .csv de textos")
                             csv_output = gr.Files(label="Arquivos .csv de resumos e avaliação")
                         csv_button = gr.Button("Executar")
                 with gr.Tab("DataFrame"):
@@ -328,12 +398,12 @@ if __name__ == "__main__":
                             interactive=False,
                         )
                         with gr.Row():
-                            df_input = gr.DataFrame(headers=["Texto","Golden Summary"],row_count=(4,"dynamic"),col_count=(2,"fixed"))
+                            df_input = gr.DataFrame(headers=["Texto","Golden Summary"],row_count=(1,"dynamic"),col_count=(2,"fixed"))
                             df_output = gr.Files(label="Arquivos .csv de resumos e avaliação")
                         df_button = gr.Button("Executar")
 
             text_button.click(run, inputs=[dropdown, text, compression_ratio, use_golden, golden], outputs=[generated_summary, evaluators])
-            csv_button.click(run_csv, inputs=[dropdown, csv_input, compression_ratio, use_golden], outputs=[csv_output])
-            df_button.click(run_csv, inputs=[dropdown, df_input, compression_ratio, use_golden], outputs=[df_output])
+            csv_button.click(run_csv, inputs=[dropdown, csv_input, text_column, compression_ratio, use_golden], outputs=[csv_output])
+            df_button.click(run_df, inputs=[dropdown, df_input, compression_ratio, use_golden], outputs=[df_output])
 
 demo.launch()