Update app.py

app.py

@@ -1,6 +1,7 @@
 # Standard library imports
 import datetime
 import base64
+import os
 
 # Related third-party imports
 import streamlit as st
@@ -10,20 +11,19 @@ from googleapiclient.discovery import build
 from dotenv import load_dotenv
 import pandas as pd
 import searchconsole
-import os
 import cohere
 from sklearn.metrics.pairwise import cosine_similarity
 import requests
 from bs4 import BeautifulSoup
 
 load_dotenv()
+
 # Initialize Cohere client
 COHERE_API_KEY = os.environ["COHERE_API_KEY"]
 co = cohere.Client(COHERE_API_KEY)
 
 # Configuration: Set to True if running locally, False if running on Streamlit Cloud
 IS_LOCAL = False
-#==TREs
 
 # Constants
 SEARCH_TYPES = ["web", "image", "video", "news", "discover", "googleNews"]
@@ -46,14 +46,9 @@ DF_PREVIEW_ROWS = 100
 # -------------
 
 def setup_streamlit():
-    """
-    Configures Streamlit's page settings and displays the app title and markdown information.
-    Sets the page layout, title, and markdown content with links and app description.
-    """
     st.set_page_config(page_title="✨ Simple Google Search Console Data | LeeFoot.co.uk", layout="wide")
     st.title("✨ Simple Google Search Console Data | June 2024")
     st.markdown(f"### Lightweight GSC Data Extractor. (Max {MAX_ROWS:,} Rows)")
-
     st.markdown(
         """
         <p>
@@ -65,10 +60,6 @@ def setup_streamlit():
     st.divider()
 
 def init_session_state():
-    """
-    Initialises or updates the Streamlit session state variables for property selection,
-    search type, date range, dimensions, and device type.
-    """
     if 'selected_property' not in st.session_state:
         st.session_state.selected_property = None
     if 'selected_search_type' not in st.session_state:
@@ -88,11 +79,11 @@ def init_session_state():
     if 'custom_end_date' not in st.session_state:
         st.session_state.custom_end_date = datetime.date.today()
 
+# -------------
+# Data Processing Functions
+# -------------
 
 def fetch_content(url):
-    """
-    Fetches the content of a webpage.
-    """
     try:
         response = requests.get(url)
         response.raise_for_status()
@@ -101,66 +92,39 @@ def fetch_content(url):
         return content
     except requests.RequestException as e:
         return str(e)
-    
+
 def generate_embeddings(text_list):
-    """
-    Generates embeddings for a list of texts using Cohere's API.
-    """
     if not text_list:
         return []
-
     model = 'embed-english-v3.0'
     input_type = 'search_document'
     response = co.embed(model=model, texts=text_list, input_type=input_type)
     embeddings = response.embeddings
     return embeddings
 
-
 def calculate_relevancy_scores(df):
-    """
-    Calculates relevancy scores for each row in the dataframe.
-    """
     try:
-        st.write("Calculating relevancy scores...")
-        st.write(f"Input DataFrame shape: {df.shape}")
-        st.write(f"Input DataFrame columns: {df.columns}")
-        
         page_contents = [fetch_content(url) for url in df['page']]
-        st.write(f"Fetched {len(page_contents)} page contents")
-        
         page_embeddings = generate_embeddings(page_contents)
-        st.write(f"Generated {len(page_embeddings)} page embeddings")
-        
         query_embeddings = generate_embeddings(df['query'].tolist())
-        st.write(f"Generated {len(query_embeddings)} query embeddings")
-        
         relevancy_scores = cosine_similarity(query_embeddings, page_embeddings).diagonal()
-        st.write(f"Calculated {len(relevancy_scores)} relevancy scores")
-        st.write(f"Sample relevancy scores: {relevancy_scores[:5]}")
-        
         df = df.assign(relevancy_score=relevancy_scores)
-        st.write(f"Assigned relevancy scores to DataFrame")
-        st.write(f"DataFrame shape after assigning scores: {df.shape}")
-        st.write(f"DataFrame columns after assigning scores: {df.columns}")
-        st.write(f"Sample relevancy scores from DataFrame: {df['relevancy_score'].head()}")
-        
     except Exception as e:
         st.warning(f"Error calculating relevancy scores: {e}")
-        df = df.assign(relevancy_score=0)  # Default value if calculation fails
-    
+        df = df.assign(relevancy_score=0)
     return df
-def fetch_data_loading(webproperty, search_type, start_date, end_date, dimensions, device_type=None):
-    """
-    Fetches Google Search Console data with a loading indicator and calculates relevancy scores.
-    """
-    with st.spinner('Fetching data and calculating relevancy scores...'):
-        df = fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, device_type)
-        if not df.empty:
-            df = calculate_relevancy_scores(df)
-        st.write(f"Data fetched. Shape: {df.shape}")
-        return df
-# -------------
 
+def process_gsc_data(df):
+    df_sorted = df.sort_values(['page', 'clicks'], ascending=[True, False])
+    df_unique = df_sorted.drop_duplicates(subset='page', keep='first').copy()
+    if 'relevancy_score' not in df_unique.columns:
+        df_unique['relevancy_score'] = 0
+    else:
+        df_unique['relevancy_score'] = df_sorted.groupby('page')['relevancy_score'].first().values
+    result = df_unique[['page', 'query', 'clicks', 'impressions', 'ctr', 'position', 'relevancy_score']]
+    return result
+
+# -------------
 # Google Authentication Functions
 # -------------
 
@@ -177,30 +141,20 @@ def load_config():
     return client_config
 
 def init_oauth_flow(client_config):
-    """
-    Initializes the OAuth flow for Google API authentication using the client configuration.
-    Sets the necessary scopes and returns the configured Flow object.
-    """
     scopes = ["https://www.googleapis.com/auth/webmasters.readonly"]
     flow = Flow.from_client_config(
         client_config,
         scopes=scopes,
-        redirect_uri=client_config["web"]["redirect_uris"][0]  # Changed from "installed" to "web"
+        redirect_uri=client_config["web"]["redirect_uris"][0]
     )
     return flow
+
 def google_auth(client_config):
-    """
-    Starts the Google authentication process using OAuth.
-    Generates and returns the OAuth flow and the authentication URL.
-    """
     flow = init_oauth_flow(client_config)
     auth_url, _ = flow.authorization_url(prompt="consent")
     return flow, auth_url
+
 def auth_search_console(client_config, credentials):
-    """
-    Authenticates the user with the Google Search Console API using provided credentials.
-    Returns an authenticated searchconsole client.
-    """
     token = {
         "token": credentials.token,
         "refresh_token": credentials.refresh_token,
@@ -217,24 +171,14 @@ def auth_search_console(client_config, credentials):
 # -------------
 
 def list_gsc_properties(credentials):
-    """
-    Lists all Google Search Console properties accessible with the given credentials.
-    Returns a list of property URLs or a message if no properties are found.
-    """
     service = build('webmasters', 'v3', credentials=credentials)
     site_list = service.sites().list().execute()
     return [site['siteUrl'] for site in site_list.get('siteEntry', [])] or ["No properties found"]
 
 def fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, device_type=None):
-    """
-    Fetches Google Search Console data for a specified property, date range, dimensions, and device type.
-    Handles errors and returns the data as a DataFrame.
-    """
     query = webproperty.query.range(start_date, end_date).search_type(search_type).dimension(*dimensions)
-
     if 'device' in dimensions and device_type and device_type != 'All Devices':
         query = query.filter('device', 'equals', device_type.lower())
-
     try:
         df = query.limit(MAX_ROWS).get().to_dataframe()
         return process_gsc_data(df)
@@ -242,88 +186,22 @@ def fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, d
         show_error(e)
         return pd.DataFrame()
 
-def process_gsc_data(df):
-    """
-    Processes the GSC data to return only unique pages with their first query and relevancy score.
-    """
-    st.write("Processing GSC data...")
-    st.write(f"Input DataFrame shape: {df.shape}")
-    st.write(f"Input DataFrame columns: {df.columns}")
-    
-    # Sort the dataframe by page and clicks (descending) to get the most relevant query first
-    df_sorted = df.sort_values(['page', 'clicks'], ascending=[True, False])
-    
-    # Get the first occurrence of each page (which will be the one with the highest clicks)
-    df_unique = df_sorted.drop_duplicates(subset='page', keep='first').copy()
-    
-    st.write(f"Unique pages DataFrame shape: {df_unique.shape}")
-    st.write(f"Unique pages DataFrame columns: {df_unique.columns}")
-    
-    # Ensure 'relevancy_score' column exists and is preserved
-    if 'relevancy_score' not in df_unique.columns:
-        st.write("Relevancy score column not found, adding default values")
-        df_unique['relevancy_score'] = 0  # Default value if column doesn't exist
-    else:
-        st.write("Preserving relevancy scores")
-        # Make sure to keep the original relevancy scores
-        df_unique['relevancy_score'] = df_sorted.groupby('page')['relevancy_score'].first().values
-    
-    # Select only the relevant columns, including the relevancy_score
-    result = df_unique[['page', 'query', 'clicks', 'impressions', 'ctr', 'position', 'relevancy_score']]
-    
-    st.write(f"Processed data. Shape: {result.shape}")
-    st.write(f"Columns: {result.columns}")
-    st.write(f"Sample relevancy scores: {result['relevancy_score'].head()}")
-    
-    return result
-
-
 def fetch_data_loading(webproperty, search_type, start_date, end_date, dimensions, device_type=None):
-    """
-    Fetches Google Search Console data with a loading indicator and calculates relevancy scores.
-    """
     with st.spinner('Fetching data and calculating relevancy scores...'):
         df = fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, device_type)
-        st.write(f"Data fetched. Shape: {df.shape}")
-        st.write(f"Columns: {df.columns}")
-        
         if not df.empty:
             df = calculate_relevancy_scores(df)
-            st.write("Relevancy scores calculated.")
-            st.write(f"DataFrame shape after calculating scores: {df.shape}")
-            st.write(f"DataFrame columns after calculating scores: {df.columns}")
-            st.write(f"Sample relevancy scores after calculation: {df['relevancy_score'].head()}")
-        
         processed_df = process_gsc_data(df)
-        st.write("Data processed.")
-        st.write(f"Final DataFrame shape: {processed_df.shape}")
-        st.write(f"Final DataFrame columns: {processed_df.columns}")
-        st.write(f"Final sample relevancy scores: {processed_df['relevancy_score'].head()}")
-        
         return processed_df
-    """
-    Fetches Google Search Console data with a loading indicator. Utilises 'fetch_gsc_data' for data retrieval.
-    Returns the fetched data as a DataFrame.
-    """
-    with st.spinner('Fetching data...'):
-        return fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, device_type)
 
 # -------------
 # Utility Functions
 # -------------
 
 def update_dimensions(selected_search_type):
-    """
-    Updates and returns the list of dimensions based on the selected search type.
-    Adds 'device' to dimensions if the search type requires it.
-    """
     return BASE_DIMENSIONS + ['device'] if selected_search_type in SEARCH_TYPES else BASE_DIMENSIONS
 
 def calc_date_range(selection, custom_start=None, custom_end=None):
-    """
-    Calculates the date range based on the selected range option.
-    Returns the start and end dates for the specified range.
-    """
     range_map = {
         'Last 7 Days': 7,
         'Last 30 Days': 30,
@@ -341,17 +219,9 @@ def calc_date_range(selection, custom_start=None, custom_end=None):
     return today - datetime.timedelta(days=range_map.get(selection, 0)), today
 
 def show_error(e):
-    """
-    Displays an error message in the Streamlit app.
-    Formats and shows the provided error 'e'.
-    """
     st.error(f"An error occurred: {e}")
 
 def property_change():
-    """
-    Updates the 'selected_property' in the Streamlit session state.
-    Triggered on change of the property selection.
-    """
     st.session_state.selected_property = st.session_state['selected_property_selector']
 
 # -------------
@@ -359,19 +229,12 @@ def property_change():
 # -------------
 
 def show_dataframe(report):
-    """
-    Shows a preview of the first 100 rows of the processed report DataFrame in an expandable section.
-    """
     with st.expander("Preview the First 100 Rows (Unique Pages with Top Query)"):
         st.dataframe(report.head(DF_PREVIEW_ROWS))
 
 def download_csv_link(report):
-    """
-    Generates and displays a download link for the report DataFrame in CSV format.
-    """
     def to_csv(df):
         return df.to_csv(index=False, encoding='utf-8-sig')
-
     csv = to_csv(report)
     b64_csv = base64.b64encode(csv.encode()).decode()
     href = f'<a href="data:file/csv;base64,{b64_csv}" download="search_console_data.csv">Download CSV File</a>'
@@ -382,20 +245,12 @@ def download_csv_link(report):
 # -------------
 
 def show_google_sign_in(auth_url):
-    """
-    Displays the Google sign-in button and authentication URL in the Streamlit sidebar.
-    """
     with st.sidebar:
         if st.button("Sign in with Google"):
-            # Open the authentication URL
             st.write('Please click the link below to sign in:')
             st.markdown(f'[Google Sign-In]({auth_url})', unsafe_allow_html=True)
 
 def show_property_selector(properties, account):
-    """
-    Displays a dropdown selector for Google Search Console properties.
-    Returns the selected property's webproperty object.
-    """
     selected_property = st.selectbox(
         "Select a Search Console Property:",
         properties,
@@ -407,10 +262,6 @@ def show_property_selector(properties, account):
     return account[selected_property]
 
 def show_search_type_selector():
-    """
-    Displays a dropdown selector for choosing the search type.
-    Returns the selected search type.
-    """
     return st.selectbox(
         "Select Search Type:",
         SEARCH_TYPES,
@@ -419,10 +270,6 @@ def show_search_type_selector():
     )
 
 def show_date_range_selector():
-    """
-    Displays a dropdown selector for choosing the date range.
-    Returns the selected date range option.
-    """
     return st.selectbox(
         "Select Date Range:",
         DATE_RANGE_OPTIONS,
@@ -431,18 +278,10 @@ def show_date_range_selector():
     )
 
 def show_custom_date_inputs():
-    """
-    Displays date input fields for custom date range selection.
-    Updates session state with the selected dates.
-    """
     st.session_state.custom_start_date = st.date_input("Start Date", st.session_state.custom_start_date)
     st.session_state.custom_end_date = st.date_input("End Date", st.session_state.custom_end_date)
 
 def show_dimensions_selector(search_type):
-    """
-    Displays a multi-select box for choosing dimensions based on the selected search type.
-    Returns the selected dimensions.
-    """
     available_dimensions = update_dimensions(search_type)
     return st.multiselect(
         "Select Dimensions:",
@@ -451,32 +290,24 @@ def show_dimensions_selector(search_type):
         key='dimensions_selector'
     )
 
-def show_fetch_data_button(webproperty, search_type, start_date, end_date, selected_dimensions):
-    """
-    Displays a button to fetch data based on selected parameters.
-    Shows the report DataFrame and download link upon successful data fetching.
-    """
-    if st.button("Fetch Data"):
-        report = fetch_data_loading(webproperty, search_type, start_date, end_date, selected_dimensions)
-
-        if report is not None and not report.empty:
-            show_dataframe(report)
-            download_csv_link(report)
-        else:
-            st.warning("No data found for the selected criteria.")
-
-
-
 def show_paginated_dataframe(report, rows_per_page=20):
-    """
-    Displays the DataFrame with custom pagination.
-    """
     total_rows = len(report)
     total_pages = (total_rows - 1) // rows_per_page + 1
 
-    page = st.number_input("Page", min_value=1, max_value=total_pages, step=1)
+    if 'current_page' not in st.session_state:
+        st.session_state.current_page = 1
 
-    start_idx = (page - 1) * rows_per_page
+    col1, col2, col3 = st.columns([1,3,1])
+    with col1:
+        if st.button("Previous", disabled=st.session_state.current_page == 1):
+            st.session_state.current_page -= 1
+    with col2:
+        st.write(f"Page {st.session_state.current_page} of {total_pages}")
+    with col3:
+        if st.button("Next", disabled=st.session_state.current_page == total_pages):
+            st.session_state.current_page += 1
+
+    start_idx = (st.session_state.current_page - 1) * rows_per_page
     end_idx = start_idx + rows_per_page
     st.dataframe(report.iloc[start_idx:end_idx])
 
@@ -487,16 +318,18 @@ def show_paginated_dataframe(report, rows_per_page=20):
 def main():
     setup_streamlit()
     client_config = load_config()
-    st.session_state.auth_flow, st.session_state.auth_url = google_auth(client_config)
+    
+    if 'auth_flow' not in st.session_state or 'auth_url' not in st.session_state:
+        st.session_state.auth_flow, st.session_state.auth_url = google_auth(client_config)
 
     query_params = st.experimental_get_query_params()
     auth_code = query_params.get("code", [None])[0]
 
-    if auth_code and not st.session_state.get('credentials'):
+    if auth_code and 'credentials' not in st.session_state:
         st.session_state.auth_flow.fetch_token(code=auth_code)
         st.session_state.credentials = st.session_state.auth_flow.credentials
 
-    if not st.session_state.get('credentials'):
+    if 'credentials' not in st.session_state:
         show_google_sign_in(st.session_state.auth_url)
     else:
         init_session_state()
@@ -516,13 +349,19 @@ def main():
 
             selected_dimensions = show_dimensions_selector(search_type)
 
-            if st.button("Fetch Data and Display"):
-                report = fetch_data_loading(webproperty, search_type, start_date, end_date, selected_dimensions)
+            if 'report_data' not in st.session_state:
+                st.session_state.report_data = None
+
+            if st.button("Fetch Data"):
+                with st.spinner('Fetching data...'):
+                    st.session_state.report_data = fetch_data_loading(webproperty, search_type, start_date, end_date, selected_dimensions)
 
-                if report is not None and not report.empty:
-                    show_paginated_dataframe(report)
-                else:
-                    st.warning("No data found for the selected criteria.")
+            if st.session_state.report_data is not None and not st.session_state.report_data.empty:
+                show_paginated_dataframe(st.session_state.report_data)
+                download_csv_link(st.session_state.report_data)
+            elif st.session_state.report_data is not None:
+                st.warning("No data found for the selected criteria.")
 
+                
 if __name__ == "__main__":
     main()