Spaces:
Running
Running
import gradio as gr | |
import os | |
import re | |
import torch | |
import pandas as pd | |
import plotly.express as px | |
import plotly.io as pio | |
import nltk | |
import tempfile | |
from io import BytesIO | |
import base64 | |
from transformers import AutoTokenizer, AutoModelForSequenceClassification | |
from nltk.tokenize import sent_tokenize | |
from docx.shared import Inches | |
from docx import Document | |
import numpy as np | |
# Needed for HF GPU access | |
import spaces | |
from styles import custom_css # Importing custom CSS | |
nltk.download('punkt') | |
# Import PyPDFLoader for PDF processing | |
from langchain_community.document_loaders import PyPDFLoader | |
# Model checkpoint for SDG BERT | |
checkpoint = "sadickam/sdgBERT" | |
# Text cleaning function | |
def clean_text(text): | |
""" | |
Cleans the extracted text by removing irrelevant characters but retains currency symbols. | |
""" | |
text = text.strip() | |
# Define the allowed characters (including currency symbols) | |
allowed_chars = r'[^a-zA-Z0-9\s\.,!?$β¬Β£Β₯βΉΒ’β©]' | |
text = re.sub(allowed_chars, '', text) | |
text = re.sub(r'\s+', ' ', text) # Replace multiple spaces with a single space | |
return text | |
# Preprocessing function for text | |
def prep_text(text): | |
clean_sents = [] | |
sent_tokens = sent_tokenize(str(text)) | |
for sent_token in sent_tokens: | |
word_tokens = [str(word_token).strip().lower() for word_token in sent_token.split()] | |
clean_sents.append(' '.join(word_tokens)) | |
joined = ' '.join(clean_sents).strip() | |
return re.sub(r'`|"', "", joined) | |
# Load the tokenizer and model with GPU support | |
def load_model_and_tokenizer(): | |
model = AutoModelForSequenceClassification.from_pretrained(checkpoint).to(device) | |
tokenizer = AutoTokenizer.from_pretrained(checkpoint) | |
return model, tokenizer | |
# Define device (ensure usage of GPU if available in Hugging Face Spaces) | |
device = torch.device("cuda" if torch.cuda.is_available() else "cpu") | |
# SDG labels | |
label_list = [ | |
'SDG1_No Poverty', 'SDG2_Zero Hunger', 'SDG3_Good Health and Well-being', 'SDG4_Quality Education', | |
'SDG5_Gender Equality', 'SDG6_Clean Water and Sanitation', 'SDG7_Affordable and Clean Energy', | |
'SDG8_Decent Work and Economic Growth', 'SDG9_Industry, Innovation and Infrastructure', | |
'SDG10_Reduced Inequality', 'SDG11_Sustainable Cities and Communities', | |
'SDG12_Responsible Consumption and Production', 'SDG13_Climate Action', | |
'SDG14_Life Below Water', 'SDG15_Life on Land', 'SDG16_Peace, Justice and Strong Institutions' | |
] | |
# Function to predict SDGs for a batch of text inputs | |
def predict_sdg_labels_batch(texts, model, tokenizer): | |
tokenized_texts = tokenizer(texts, return_tensors="pt", truncation=True, padding=True, max_length=512).to(device) | |
model.eval() | |
with torch.no_grad(): | |
text_logits = model(**tokenized_texts).logits | |
predictions = torch.softmax(text_logits, dim=1).tolist() | |
return predictions | |
# Page-level predictions with batch processing | |
def predict_pages(page_df, batch_size=32): | |
model, tokenizer = load_model_and_tokenizer() | |
df_results = page_df.copy() | |
num_rows = len(page_df) | |
all_predicted_labels = [[] for _ in range(16)] | |
all_prediction_scores = [[] for _ in range(16)] | |
for start in range(0, num_rows, batch_size): | |
end = min(start + batch_size, num_rows) | |
df_chunk = page_df.iloc[start:end] | |
# Clean text | |
texts = df_chunk['Text'].apply(clean_text).apply(prep_text).tolist() | |
predictions_batch = predict_sdg_labels_batch(texts, model, tokenizer) | |
for predictions in predictions_batch: | |
sorted_preds = sorted(zip(label_list, predictions), key=lambda x: x[1], reverse=True) | |
for i, (label, score) in enumerate(sorted_preds): | |
all_predicted_labels[i].append(label) | |
all_prediction_scores[i].append(score) | |
# Add columns to the DataFrame in the desired order (pred1, score1, pred2, score2, ...) | |
for i in range(16): | |
df_results[f'pred{i + 1}'] = all_predicted_labels[i] | |
df_results[f'score{i + 1}'] = all_prediction_scores[i] | |
# Reorder columns to ensure preds and scores are interleaved in the correct order | |
reordered_columns = [] | |
for i in range(16): | |
reordered_columns.append(f'pred{i + 1}') | |
reordered_columns.append(f'score{i + 1}') | |
other_columns = [col for col in df_results.columns if col not in reordered_columns] | |
df_results = df_results[other_columns + reordered_columns] | |
return df_results | |
# Sentence-level predictions with batch processing | |
def predict_sentences(sentence_df, batch_size=32): | |
model, tokenizer = load_model_and_tokenizer() | |
df_combined_sentences = sentence_df.copy() | |
num_rows = len(sentence_df) | |
all_predicted_labels = [[] for _ in range(16)] | |
all_prediction_scores = [[] for _ in range(16)] | |
for start in range(0, num_rows, batch_size): | |
end = min(start + batch_size, num_rows) | |
df_chunk = sentence_df.iloc[start:end] | |
# Clean text | |
texts = df_chunk['Sentence'].apply(clean_text).apply(prep_text).tolist() | |
predictions_batch = predict_sdg_labels_batch(texts, model, tokenizer) | |
for predictions in predictions_batch: | |
sorted_preds = sorted(zip(label_list, predictions), key=lambda x: x[1], reverse=True) | |
for i, (label, score) in enumerate(sorted_preds): | |
all_predicted_labels[i].append(label) | |
all_prediction_scores[i].append(round(score, 3)) | |
# Add predictions and scores to DataFrame | |
for i in range(16): | |
df_combined_sentences[f'pred{i + 1}'] = all_predicted_labels[i] | |
df_combined_sentences[f'score{i + 1}'] = all_prediction_scores[i] | |
# Reorder columns | |
reordered_columns = [] | |
for i in range(16): | |
reordered_columns.append(f'pred{i + 1}') | |
reordered_columns.append(f'score{i + 1}') | |
other_columns = [col for col in df_combined_sentences.columns if col not in reordered_columns] | |
df_combined_sentences = df_combined_sentences[other_columns + reordered_columns] | |
return df_combined_sentences | |
# Define unique colors for each SDG | |
sdg_colors = { | |
"SDG1_No Poverty": "#E5243B", | |
"SDG2_Zero Hunger": "#DDA63A", | |
"SDG3_Good Health and Well-being": "#4C9F38", | |
"SDG4_Quality Education": "#C5192D", | |
"SDG5_Gender Equality": "#FF3A21", | |
"SDG6_Clean Water and Sanitation": "#26BDE2", | |
"SDG7_Affordable and Clean Energy": "#FCC30B", | |
"SDG8_Decent Work and Economic Growth": "#A21942", | |
"SDG9_Industry, Innovation and Infrastructure": "#FD6925", | |
"SDG10_Reduced Inequality": "#DD1367", | |
"SDG11_Sustainable Cities and Communities": "#FD9D24", | |
"SDG12_Responsible Consumption and Production": "#BF8B2E", | |
"SDG13_Climate Action": "#3F7E44", | |
"SDG14_Life Below Water": "#0A97D9", | |
"SDG15_Life on Land": "#56C02B", | |
"SDG16_Peace, Justice and Strong Institutions": "#00689D" | |
} | |
# Function to plot SDG dominant bar graphs using Plotly | |
# Function to plot SDG dominant bar graphs using Plotly | |
def plot_sdg(df, title, pred_column, x_axis_title=None, y_axis_title=None, icons_folder='assets/icons/'): | |
""" | |
Plots a horizontal bar graph of SDG predictions and superimposes the icon of the most frequent SDG. | |
Args: | |
df (pd.DataFrame): DataFrame containing SDG predictions. | |
title (str): Title of the plot. | |
pred_column (str): Column name to use for plotting (e.g., 'pred1'). | |
x_axis_title (str): Title for the x-axis. | |
y_axis_title (str): Title for the y-axis. | |
icons_folder (str): Path to the folder containing SDG icons. | |
Returns: | |
plotly.graph_objs._figure.Figure: The Plotly figure object. | |
""" | |
df_filtered = df[df[pred_column].notna()] | |
labels = df_filtered[pred_column].value_counts().sort_values(ascending=False) | |
total = labels.sum() | |
percentages = (labels / total) * 100 | |
# Create a horizontal bar plot with Plotly | |
fig = px.bar( | |
percentages.rename_axis('SDG Label').reset_index(name='Percentage'), | |
y='SDG Label', | |
x='Percentage', | |
orientation='h', | |
title=title, | |
color='SDG Label', | |
color_discrete_map=sdg_colors # Use the defined unique colors for each SDG | |
) | |
# Update y-axis to show labels | |
fig.update_yaxes(showticklabels=True) | |
# Add percentage labels to the bars | |
fig.update_traces( | |
texttemplate='%{x:.2f}%', | |
textposition='auto', | |
textfont=dict(size=11) | |
) | |
# Adjust layout for better visibility | |
fig.update_layout( | |
title=dict( | |
text=title, font=dict(size=14) # Increase title font size | |
), | |
yaxis=dict( | |
automargin=True, | |
title=y_axis_title, | |
tickfont=dict(size=13) | |
), | |
margin=dict(l=20, r=100, t=30, b=20), # Increased right margin for icon | |
height=600, | |
#width=800, | |
showlegend=False, | |
template="simple_white", | |
xaxis=dict( | |
title=x_axis_title, | |
tickfont=dict(size=13) # Reduce x-axis font size | |
), | |
) | |
# Identify the most frequent SDG | |
if not percentages.empty: | |
top_sdg_label = percentages.index[0] # e.g., 'SDG1_No Poverty' | |
# Map SDG label to icon filename | |
# Assuming naming convention 'SDG1.png', 'SDG2.png', etc. | |
sdg_number = top_sdg_label.split('_')[0] # Extract 'SDG1' | |
icon_filename = f"{sdg_number}.png" # e.g., 'SDG1.png' | |
icon_path = os.path.join(icons_folder, icon_filename) | |
# Check if the icon file exists | |
if os.path.exists(icon_path): | |
# Read and encode the image | |
with open(icon_path, 'rb') as image_file: | |
encoded_image = base64.b64encode(image_file.read()).decode('utf-8') | |
# Add the icon as an image in the Plotly figure | |
fig.add_layout_image( | |
dict( | |
source='data:image/png;base64,' + encoded_image, | |
xref="paper", yref="paper", | |
x=.98, y=1.0, # Positioning: slightly to the right and top | |
sizex=0.2, sizey=0.2, # Size of the icon | |
xanchor="left", | |
yanchor="top", | |
layer="above" # Ensure the icon is above other plot elements | |
) | |
) | |
else: | |
print(f"Icon file '{icon_path}' not found. Skipping icon overlay.") | |
return fig | |
def save_figure_as_jpeg(fig, filename): | |
"""Saves the Plotly figure as a high-resolution JPEG.""" | |
pio.write_image(fig, filename, format='jpeg', width=700, height=650, scale=7, engine="kaleido") | |
# Generate reports (page and sentence levels) | |
def generate_page_report(df_pages, report_file_name): | |
doc = Document() | |
doc.add_heading("Page-Level SDG Analysis Report", 0) | |
doc.add_heading("π General Notes", level=2) | |
doc.add_paragraph( | |
'This app conducts page-level analysis of the uploaded document. Each page is processed by the sdgBERT AI model trained to predict the first 16 ' | |
'Sustainable Development Goals (SDGs). The model analyzes the content and returns scores ' | |
'representing the likelihood that the text is aligned with particular SDGs. This page-level ' | |
'analysis provides high-level insight into SDG alignment.' | |
'\n\n' | |
'Given that a page may align with more than one SDG, this app focuses on the top two SDG predictions ' | |
'(Primary and Secondary) for each page with a probability score greater than zero.' | |
) | |
doc.add_heading("Primary SDGs Bar Graph", level=3) | |
doc.add_paragraph( | |
'This graph displays the most essential SDG the AI model associates with pages. The bars ' | |
'represent the percentage of pages most strongly aligned with each SDG. This offers insight into the dominant ' | |
'sustainable development theme within the document.' | |
) | |
doc.add_heading("Secondary SDGs Bar Graph", level=3) | |
doc.add_paragraph( | |
'This graph shows the second most relevant SDGs for pages. Although these SDGs are ' | |
'not the primary focus, the text has some relevance to these goals.' | |
) | |
for doc_name in df_pages['Document'].unique(): | |
# Sanitize doc_name to use in file names | |
sanitized_doc_name = re.sub(r'[^\w\-]', '_', os.path.splitext(doc_name)[0]) | |
doc.add_heading(f"π Document: {doc_name}", level=2) | |
df_doc = df_pages[df_pages['Document'] == doc_name] | |
# Generate and save graphs | |
first_sdg_plot_path = f"{sanitized_doc_name}_first_sdg_page.jpeg" | |
second_sdg_plot_path = f"{sanitized_doc_name}_second_sdg_page.jpeg" | |
plot_sdg( | |
df_doc, "", 'pred1', | |
x_axis_title="Percentage (%) of aligned pages", | |
y_axis_title="Primary SDGs" | |
).write_image(first_sdg_plot_path, format='jpeg', scale=7, engine="kaleido") | |
plot_sdg( | |
df_doc, "", 'pred2', | |
x_axis_title="Percentage (%) of aligned pages", | |
y_axis_title="Secondary SDGs" | |
).write_image(second_sdg_plot_path, format='jpeg', scale=7, engine="kaleido") | |
# Add plots to the Word document | |
doc.add_picture(first_sdg_plot_path, width=Inches(6)) | |
doc.add_picture(second_sdg_plot_path, width=Inches(6)) | |
doc.save(report_file_name) | |
return report_file_name | |
def generate_sentence_report(df_sentences, report_file_name): | |
doc = Document() | |
doc.add_heading("Sentence-Level SDG Analysis Report", 0) | |
doc.add_heading("π General Notes", level=2) | |
doc.add_paragraph( | |
'This app splits documents into sentences using a natural language processing algorithm. ' | |
'Each sentence is processed by the sdgBERT AI model trained to predict the first 16 ' | |
'Sustainable Development Goals (SDGs). The model analyzes the content and returns scores ' | |
'representing the likelihood that the text is aligned with particular SDGs. This sentence-level ' | |
'analysis provides deeper insight into SDG alignment.' | |
'\n\n' | |
'Given that a sentence may align with more than one SDG, this app focuses on the top two SDG predictions ' | |
'(Primary and Secondary) for each sentence with a probability score greater than zero.' | |
) | |
doc.add_heading("Primary SDGs Bar Graph", level=3) | |
doc.add_paragraph( | |
'This graph displays the most essential SDG the AI model associates with sentences. The bars ' | |
'represent the percentage of sentences most strongly aligned with each SDG. This offers more profound insight ' | |
'into the dominant sustainable development theme within the document.' | |
) | |
doc.add_heading("Secondary SDGs Bar Graph", level=3) | |
doc.add_paragraph( | |
'This graph shows the second most relevant SDGs for sentences. Although these SDGs are not ' | |
'the primary focus, the text has some relevance to these goals.' | |
) | |
for doc_name in df_sentences['Document'].unique(): | |
# Sanitize doc_name to use in file names | |
sanitized_doc_name = re.sub(r'[^\w\-]', '_', os.path.splitext(doc_name)[0]) | |
doc.add_heading(f"π Document: {doc_name}", level=2) | |
df_doc = df_sentences[df_sentences['Document'] == doc_name] | |
# Generate and save graphs | |
first_sdg_plot_path = f"{sanitized_doc_name}_first_sdg_sentence.jpeg" | |
second_sdg_plot_path = f"{sanitized_doc_name}_second_sdg_sentence.jpeg" | |
plot_sdg( | |
df_doc, "", 'pred1', | |
x_axis_title="Percentage (%) of aligned sentences", | |
y_axis_title="Primary SDGs" | |
).write_image(first_sdg_plot_path, format='jpeg', scale=7, engine="kaleido") | |
plot_sdg( | |
df_doc, "", 'pred2', | |
x_axis_title="Percentage (%) of aligned sentences", | |
y_axis_title="Secondary SDGs" | |
).write_image(second_sdg_plot_path, format='jpeg', scale=7, engine="kaleido") | |
# Add plots to the Word document | |
doc.add_picture(first_sdg_plot_path, width=Inches(6)) | |
doc.add_picture(second_sdg_plot_path, width=Inches(6)) | |
doc.save(report_file_name) | |
return report_file_name | |
# New text extraction functions with text cleaning and line joining | |
def extract_text_with_py_pdf_loader(pdf_file_path, start_page=None, end_page=None): | |
""" | |
Extract text from a PDF page by page using LangChain's PyPDFLoader. | |
Args: | |
pdf_file_path (str): The file path to the uploaded PDF. | |
start_page (int, optional): The starting page number for extraction (1-based index). | |
end_page (int, optional): The ending page number for extraction (1-based index). | |
Returns: | |
tuple: | |
- page_df (pd.DataFrame): DataFrame containing Document, Page, and Text. | |
- sentence_df (pd.DataFrame): DataFrame containing Document, Page, and Sentence. | |
""" | |
try: | |
# Initialize the loader | |
loader = PyPDFLoader(pdf_file_path) | |
documents = loader.load_and_split() # Each document corresponds to a single page | |
total_pages = len(documents) | |
doc_name = os.path.basename(pdf_file_path) # Extract document name | |
# Validate and adjust page range | |
if start_page is not None and end_page is not None: | |
# Convert to integers to avoid slicing issues | |
start_page = int(start_page) | |
end_page = int(end_page) | |
# Adjust to valid range | |
if start_page < 1: | |
start_page = 1 | |
if end_page > total_pages: | |
end_page = total_pages | |
if start_page > end_page: | |
start_page, end_page = end_page, start_page # Swap if out of order | |
# Select the subset of documents based on user input | |
selected_docs = documents[start_page - 1:end_page] | |
else: | |
selected_docs = documents | |
start_page = 1 | |
end_page = total_pages | |
# Initialize lists to store data | |
page_data = [] | |
sentence_data = [] | |
for idx, doc in enumerate(selected_docs, start=start_page): | |
page_num = idx | |
text = doc.page_content.strip() | |
# Join lines that belong to the same sentence | |
lines = text.split('\n') | |
joined_text = ' '.join(line.strip() for line in lines if line.strip()) | |
# Clean text | |
cleaned_text = clean_text(joined_text) | |
# Append page-wise data | |
page_data.append({ | |
"Document": doc_name, | |
"Page": page_num, | |
"Text": cleaned_text | |
}) | |
# Sentence tokenization | |
sentences = sent_tokenize(cleaned_text) | |
for sentence in sentences: | |
sentence = sentence.strip() | |
if sentence and len(sentence) > 70: | |
sentence_data.append({ | |
"Document": doc_name, | |
"Page": page_num, | |
"Sentence": sentence | |
}) | |
# Create DataFrames | |
page_df = pd.DataFrame(page_data) | |
sentence_df = pd.DataFrame(sentence_data) | |
return page_df, sentence_df | |
except Exception as e: | |
raise RuntimeError(f"Error during PDF extraction: {e}") | |
def df_to_csv_bytes(df): | |
""" | |
Convert DataFrame to CSV in bytes. | |
Args: | |
df (pd.DataFrame): The DataFrame to convert. | |
Returns: | |
bytes: CSV data in bytes. | |
""" | |
try: | |
buffer = BytesIO() | |
df.to_csv(buffer, index=False) | |
csv_data = buffer.getvalue() | |
buffer.close() | |
return csv_data | |
except Exception as e: | |
raise RuntimeError(f"Error during CSV conversion: {e}") | |
def launch_interface(): | |
with gr.Blocks(css=custom_css) as demo: | |
# Title as a visible heading at the top of the page with an icon | |
gr.Markdown( | |
""" | |
# π SDG Document Analysis App | |
Analyze documents to map Sustainable Development Goals (SDGs) at both page and sentence levels. | |
""" | |
) | |
# Shared PDF file input for both analyses | |
gr.Markdown("## Upload PDF File") | |
with gr.Row(): | |
file_input = gr.File( | |
label="π Upload PDF File for Analysis", file_types=[".pdf"] | |
) | |
# Extraction mode selection with explanatory text | |
gr.Markdown( | |
""" | |
## PDF Text Extraction Mode | |
Choose whether to analyze all pages or a specific range of pages. If you want to exclude certain pages from the analysis, select | |
"Range of Pages" and specify the start and end pages. | |
""" | |
) | |
with gr.Row(): | |
extraction_mode = gr.Radio( | |
choices=["All Pages", "Range of Pages"], | |
value="All Pages", | |
label="Extraction Mode" | |
) | |
with gr.Row(): | |
start_page = gr.Number(value=1, label="π’ Start Page", visible=False, info="The cover page is page 1") | |
end_page = gr.Number(value=1, label="π’ End Page", visible=False) | |
# Function to update visibility of start_page and end_page | |
def update_page_inputs(extraction_mode): | |
if extraction_mode == "Range of Pages": | |
return gr.update(visible=True), gr.update(visible=True) | |
else: | |
return gr.update(visible=False), gr.update(visible=False) | |
extraction_mode.change( | |
update_page_inputs, | |
inputs=extraction_mode, | |
outputs=[start_page, end_page] | |
) | |
# Main Tabs for Page-Level and Sentence-Level Analysis | |
gr.Markdown("## SDG Analysis Type") | |
with gr.Tab("π Page-Level Analysis"): | |
gr.Markdown( | |
""" | |
### Page-Level SDG Analysis | |
This section conducts Sustainable Development Goals (SDG) mapping | |
of documents using the [sdgBERT model](https://huggingface.co/sadickam/sdgBERT). | |
It provides **high-level SDG mapping** of documents at the page level. | |
""" | |
) | |
with gr.Row(): | |
page_button = gr.Button("πββοΈ Run Page-Level Analysis") | |
reset_page_button = gr.Button("π Reset Page-Level Analysis", elem_classes="reset-button") | |
# Nested Tabs for Primary and Secondary SDGs | |
with gr.Tabs(): | |
with gr.TabItem("π Primary SDGs"): | |
with gr.Row(): | |
primary_page_plot = gr.Plot(label="π Primary SDGs Graph [Page-Level]", scale=2) | |
gr.Markdown( | |
"When the analysis is done, the Primary SDGs bar graph on the left will show "+ | |
"the percentage of pages that strongly align with each SDG. The icon for the most frequent "+ | |
"SDG will be highlighted on the graph. Download the Page Predictions CSV for further details.", | |
label = "Note", container=True | |
) | |
gr.Markdown("##### Download Results") | |
with gr.Row(): | |
page_csv = gr.File(label="π Download Page Predictions CSV") | |
page_docx = gr.File(label="π Download Page Report DOCX") | |
page_jpeg1 = gr.File(label="πΌοΈ Download Primary SDGs JPEG") | |
with gr.TabItem("π Secondary SDGs"): | |
with gr.Row(): | |
secondary_page_plot = gr.Plot(label="π Secondary SDGs Graph [Page-Level]", scale=2) | |
gr.Markdown( | |
"When the analysis is done, the Secondary SDGs bar graph on the left will show "+ | |
"SDGs that are not the primary focus of the pages analysed. These SDGs are second to the "+ | |
"Primary SDGs. Download the Sentence Predictions CSV for further details", | |
label = "Note", container=True | |
) | |
gr.Markdown("##### Download Results") | |
with gr.Row(): | |
page_csv_secondary = gr.File(label="π Download Page Predictions CSV") | |
page_report_file_secondary = gr.File(label="π Download Page Report DOCX") | |
secondary_page_jpeg = gr.File(label="πΌοΈ Download Secondary SDGs JPEG") | |
with gr.Tab("βοΈ Sentence-Level Analysis"): | |
gr.Markdown( | |
""" | |
### Sentence-Level SDG Analysis | |
This section conducts Sustainable Development Goals (SDG) mapping | |
using the [sdgBERT model](https://huggingface.co/sadickam/sdgBERT). | |
It provides **detailed SDG mapping** at the sentence level. | |
""" | |
) | |
with gr.Row(): | |
sentence_button = gr.Button("πββοΈ Run Sentence-Level Analysis") | |
reset_sentence_button = gr.Button("π Reset Sentence-Level Analysis", elem_classes="reset-button") | |
# Nested Tabs for Primary and Secondary SDGs | |
with gr.Tabs(): | |
with gr.TabItem("π Primary SDGs"): | |
with gr.Row(): | |
primary_sentence_plot = gr.Plot(label="π Primary SDGs Graph [Sentence-Level]", scale=2) | |
gr.Markdown( | |
"When the analysis is done, the Primary SDGs bar graph on the left will show "+ | |
"the percentage of sentences that strongly align with each SDG. The icon for the most frequent "+ | |
"SDG will be highlighted on the graph. Download the Sentence Predictions CSV for further details.", | |
label = "Note", container=True | |
) | |
gr.Markdown("##### Download Results") | |
with gr.Row(): | |
sentence_csv = gr.File(label="π Download Sentence Predictions CSV") | |
sentence_docx = gr.File(label="π Download Sentence Report DOCX") | |
sentence_jpeg1 = gr.File(label="πΌοΈ Download Primary SDGs JPEG") | |
with gr.TabItem("π Secondary SDGs"): | |
with gr.Row(): | |
secondary_sentence_plot = gr.Plot(label="π Secondary SDGs Graph [Sentence-Level]", scale=2) | |
gr.Markdown( | |
"When the analysis is done, the Secondary SDGs bar graph on the left will show "+ | |
"SDGs that are not the primary focus of the sentences analysed. These SDGs are second to the "+ | |
"Primary SDGs. Download the Sentence Predictions CSV for further details", | |
label = "Note", container=True | |
) | |
gr.Markdown("##### Download Results") | |
with gr.Row(): | |
sentence_csv_secondary = gr.File(label="π Download Sentence Predictions CSV") | |
sentence_report_file_secondary = gr.File(label="π Download Sentence Report DOCX") | |
secondary_sentence_jpeg = gr.File(label="πΌοΈ Download Secondary SDGs JPEG") | |
# Function to process page-level analysis | |
def process_pages(file, extraction_mode, start_page, end_page): | |
if not file: | |
# Return None for each output component | |
return [None, None, None, None, None, None, None, None] | |
try: | |
if hasattr(file, 'name'): | |
pdf_file_path = file.name | |
original_file_name = os.path.basename(file.name) | |
else: | |
# Save the file to a temporary location | |
with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_pdf: | |
temp_pdf.write(file.read()) | |
pdf_file_path = temp_pdf.name | |
original_file_name = 'uploaded_document' | |
# Sanitize the file name to use in output file names | |
sanitized_file_name = os.path.splitext(original_file_name)[0] | |
sanitized_file_name = re.sub(r'[^\w\-]', '_', sanitized_file_name) | |
# Determine page range based on extraction_mode | |
if extraction_mode == "All Pages": | |
selected_start = None | |
selected_end = None | |
else: | |
selected_start = int(start_page) | |
selected_end = int(end_page) | |
# Extract text and create DataFrames | |
page_df, _ = extract_text_with_py_pdf_loader( | |
pdf_file_path, | |
start_page=selected_start, | |
end_page=selected_end | |
) | |
# Predict SDGs at page level | |
df_page_predictions = predict_pages(page_df) | |
# Generate plots with icon overlay | |
first_plot = plot_sdg( | |
df_page_predictions, "", | |
'pred1', | |
x_axis_title="Percentage (%) of aligned pages", | |
y_axis_title="Primary SDGs" | |
) | |
second_plot = plot_sdg( | |
df_page_predictions, "", | |
'pred2', | |
x_axis_title="Percentage (%) of aligned pages", | |
y_axis_title="Secondary SDGs" | |
) | |
# Define output file names | |
page_csv_file = f"{sanitized_file_name}_SDG-Page_predictions.csv" | |
page_report_file = f"{sanitized_file_name}_SDG-Page_report.docx" | |
primary_page_jpeg = f"{sanitized_file_name}_SDG-Page_primary_graph.jpeg" | |
page_csv_file_secondary = f"{sanitized_file_name}_SDG-Page_predictions.csv" | |
page_report_file_secondary = f"{sanitized_file_name}_SDG-Page_report.docx" | |
secondary_page_jpeg = f"{sanitized_file_name}_SDG-Page_secondary_graph.jpeg" | |
# Save CSV and reports | |
df_page_predictions.to_csv(page_csv_file, index=False) | |
page_report_primary = generate_page_report(df_page_predictions, page_report_file) | |
df_page_predictions.to_csv(page_csv_file_secondary, index=False) | |
page_report_secondary = generate_page_report(df_page_predictions, page_report_file_secondary) | |
# Save figures as JPEG | |
save_figure_as_jpeg(first_plot, primary_page_jpeg) | |
save_figure_as_jpeg(second_plot, secondary_page_jpeg) | |
return ( | |
first_plot, second_plot, | |
page_csv_file, page_report_file, primary_page_jpeg, | |
page_csv_file_secondary, page_report_file_secondary, secondary_page_jpeg | |
) | |
except Exception as e: | |
print(f"Error: {e}") | |
return [None, None, None, None, None, None, None, None] | |
# Function to process sentence-level analysis | |
def process_sentences(file, extraction_mode, start_page, end_page): | |
if not file: | |
# Return None for each output component | |
return [None, None, None, None, None, None, None, None] | |
try: | |
if hasattr(file, 'name'): | |
pdf_file_path = file.name | |
original_file_name = os.path.basename(file.name) | |
else: | |
# Save the file to a temporary location | |
with tempfile.NamedTemporaryFile(delete=False, suffix='.pdf') as temp_pdf: | |
temp_pdf.write(file.read()) | |
pdf_file_path = temp_pdf.name | |
original_file_name = 'uploaded_document' | |
# Sanitize the file name to use in output file names | |
sanitized_file_name = os.path.splitext(original_file_name)[0] | |
sanitized_file_name = re.sub(r'[^\w\-]', '_', sanitized_file_name) | |
# Determine page range based on extraction_mode | |
if extraction_mode == "All Pages": | |
selected_start = None | |
selected_end = None | |
else: | |
selected_start = int(start_page) | |
selected_end = int(end_page) | |
# Extract text and create DataFrames | |
_, sentence_df = extract_text_with_py_pdf_loader( | |
pdf_file_path, | |
start_page=selected_start, | |
end_page=selected_end | |
) | |
# Predict SDGs at sentence level | |
df_sentence_predictions = predict_sentences(sentence_df) | |
# Generate plots with icon overlay | |
first_plot = plot_sdg( | |
df_sentence_predictions, "", | |
'pred1', | |
x_axis_title="Percentage (%) of aligned sentences", | |
y_axis_title="Primary SDGs" | |
) | |
second_plot = plot_sdg( | |
df_sentence_predictions, "", | |
'pred2', | |
x_axis_title="Percentage (%) of aligned sentences", | |
y_axis_title="Secondary SDGs" | |
) | |
# Define output file names | |
sentence_csv_file = f"{sanitized_file_name}_SDG-Sentence_predictions.csv" | |
sentence_report_file = f"{sanitized_file_name}_SDG-Sentence_report.docx" | |
primary_sentence_jpeg = f"{sanitized_file_name}_SDG-Sentence_primary_graph.jpeg" | |
sentence_csv_file_secondary = f"{sanitized_file_name}_SDG-Sentence_predictions.csv" | |
sentence_report_file_secondary = f"{sanitized_file_name}_SDG-Sentence_report.docx" | |
secondary_sentence_jpeg = f"{sanitized_file_name}_SDG-Sentence_secondary_graph.jpeg" | |
# Save CSV and reports | |
df_sentence_predictions.to_csv(sentence_csv_file, index=False) | |
sentence_report_primary = generate_sentence_report(df_sentence_predictions, sentence_report_file) | |
df_sentence_predictions.to_csv(sentence_csv_file_secondary, index=False) | |
sentence_report_secondary = generate_sentence_report(df_sentence_predictions, sentence_report_file_secondary) | |
# Save figures as JPEG | |
save_figure_as_jpeg(first_plot, primary_sentence_jpeg) | |
save_figure_as_jpeg(second_plot, secondary_sentence_jpeg) | |
return ( | |
first_plot, second_plot, | |
sentence_csv_file, sentence_report_file, primary_sentence_jpeg, | |
sentence_csv_file_secondary, sentence_report_file_secondary, secondary_sentence_jpeg | |
) | |
except Exception as e: | |
print(f"Error: {e}") | |
return [None, None, None, None, None, None, None, None] | |
# Reset functions to clear the outputs | |
def reset_page_outputs(): | |
return [None, None, None, None, None, None, None, None] | |
def reset_sentence_outputs(): | |
return [None, None, None, None, None, None, None, None] | |
# Button actions for Page-Level Analysis | |
page_button.click( | |
process_pages, | |
inputs=[file_input, extraction_mode, start_page, end_page], | |
outputs=[ | |
primary_page_plot, # π Primary SDGs [Page-Level] | |
secondary_page_plot, # π Secondary SDGs [Page-Level] | |
page_csv, # π Download Page Predictions CSV | |
page_docx, # π Download Page Report DOCX | |
page_jpeg1, # πΌοΈ Download Primary SDGs JPEG | |
page_csv_secondary, # π Download Page Predictions CSV | |
page_report_file_secondary, # π Download Page Report DOCX | |
secondary_page_jpeg # πΌοΈ Download Secondary SDGs JPEG | |
] | |
) | |
reset_page_button.click( | |
reset_page_outputs, | |
outputs=[ | |
primary_page_plot, | |
secondary_page_plot, | |
page_csv, | |
page_docx, | |
page_jpeg1, | |
page_csv_secondary, | |
page_report_file_secondary, | |
secondary_page_jpeg | |
] | |
) | |
# Button actions for Sentence-Level Analysis | |
sentence_button.click( | |
process_sentences, | |
inputs=[file_input, extraction_mode, start_page, end_page], | |
outputs=[ | |
primary_sentence_plot, # π Primary SDGs [Sentence-Level] | |
secondary_sentence_plot, # π Secondary SDGs [Sentence-Level] | |
sentence_csv, # π Download Sentence Predictions CSV | |
sentence_docx, # π Download Sentence Report DOCX | |
sentence_jpeg1, # πΌοΈ Download Primary SDGs JPEG | |
sentence_csv_secondary, # π Download Sentence Predictions CSV | |
sentence_report_file_secondary, # π Download Sentence Report DOCX | |
secondary_sentence_jpeg # πΌοΈ Download Secondary SDGs JPEG | |
] | |
) | |
reset_sentence_button.click( | |
reset_sentence_outputs, | |
outputs=[ | |
primary_sentence_plot, | |
secondary_sentence_plot, | |
sentence_csv, | |
sentence_docx, | |
sentence_jpeg1, | |
sentence_csv_secondary, | |
sentence_report_file_secondary, | |
secondary_sentence_jpeg | |
] | |
) | |
demo.queue().launch() | |
launch_interface() |