Update app.py

app.py

@@ -1,5 +1,3 @@
-import logging
-
 # Standard library imports
 import datetime
 import base64
@@ -20,26 +18,20 @@ from bs4 import BeautifulSoup
 from apify_client import ApifyClient
 import urllib.parse
 
-# Configure logging
-logging.basicConfig(level=logging.DEBUG, format='%(asctime)s - %(levelname)s - %(message)s')
 
 load_dotenv()
-logging.info("Environment variables loaded")
-logger = logging.getLogger(__name__)
+
 
 # Initialize Cohere client
 APIFY_API_TOKEN = os.environ.get('APIFY_API_TOKEN')
 COHERE_API_KEY = os.environ["COHERE_API_KEY"]
 co = cohere.Client(COHERE_API_KEY)
-logging.info("Cohere client initialized")
 if not APIFY_API_TOKEN:
-    logger.error("APIFY_API_TOKEN is not set in the environment variables.")
     st.error("APIFY_API_TOKEN is not set in the environment variables. Please set it and restart the application.")
 
 # Initialize the ApifyClient with the API token
 client = ApifyClient(APIFY_API_TOKEN)
 # Initialize the ApifyClient with the API token
-logger.info("ApifyClient initialized")
 
 # Configuration: Set to True if running locally, False if running on Streamlit Cloud
 IS_LOCAL = False
@@ -89,11 +81,23 @@ def init_session_state():
         st.session_state.custom_start_date = datetime.date.today() - datetime.timedelta(days=7)
     if 'custom_end_date' not in st.session_state:
         st.session_state.custom_end_date = datetime.date.today()
-    logging.info("Session state initialized")
+    #logging.info("Session state initialized")
 
 # -------------
 # Data Processing Functions
 # -------------
+def generate_embeddings(text_list, model_type):
+    #logging.debug(f"Generating embeddings for model type: {model_type}")
+    if not text_list:
+        logging.warning("Text list is empty, returning empty embeddings")
+        return []
+    model = 'embed-english-v3.0' if model_type == 'english' else 'embed-multilingual-v3.0'
+    input_type = 'search_document'
+    response = co.embed(model=model, texts=text_list, input_type=input_type)
+    embeddings = response.embeddings
+   # logging.debug(f"Embeddings generated successfully for model type: {model_type}")
+    return embeddings
+
 
 def get_serp_results(query):
     if not APIFY_API_TOKEN:
@@ -116,24 +120,24 @@ def get_serp_results(query):
         #logger.debug(f"Calling Apify Actor with input: {run_input}")
         # Run the Actor and wait for it to finish
         run = client.actor("nFJndFXA5zjCTuudP").call(run_input=run_input)
-        logger.info(f"Apify Actor run completed. Run ID: {run.get('id')}")
+       # logger.info(f"Apify Actor run completed. Run ID: {run.get('id')}")
         
         # Fetch results from the run's dataset
         
         #logger.debug(f"Fetching results from dataset ID: {run.get('defaultDatasetId')}")
         results = list(client.dataset(run["defaultDatasetId"]).iterate_items())
-        logger.info(f"Fetched {len(results)} results from Apify dataset")
+       # logger.info(f"Fetched {len(results)} results from Apify dataset")
         
         if results and 'organicResults' in results[0]:
             urls = [item['url'] for item in results[0]['organicResults']]
-            logger.info(f"Extracted {len(urls)} URLs from organic results")
+           # logger.info(f"Extracted {len(urls)} URLs from organic results")
             return urls
         else:
-            logger.warning("No organic results found in the SERP data.")
+           # logger.warning("No organic results found in the SERP data.")
             st.warning("No organic results found in the SERP data.")
             return []
     except Exception as e:
-        logger.exception(f"Error fetching SERP results: {str(e)}")
+       # logger.exception(f"Error fetching SERP results: {str(e)}")
         st.error(f"Error fetching SERP results: {str(e)}")
         return []
 
@@ -141,7 +145,7 @@ def get_serp_results(query):
 
         
 def fetch_content(url):
-    logger.info(f"Fetching content from URL: {url}")
+   # logger.info(f"Fetching content from URL: {url}")
     try:
         # Decode URL-encoded characters
         decoded_url = urllib.parse.unquote(url)
@@ -152,29 +156,29 @@ def fetch_content(url):
         #logger.debug(f"Fetched {len(content)} characters from {url}")
         return content
     except requests.RequestException as e:
-        logger.error(f"Error fetching content from {url}: {e}")
+       # logger.error(f"Error fetching content from {url}: {e}")
         st.warning(f"Error fetching content from {url}: {e}")
         return ""
 
 def calculate_relevance_score(page_content, query, co):
-    logger.info(f"Calculating relevance score for query: {query}")
+   # logger.info(f"Calculating relevance score for query: {query}")
     try:
         if not page_content:
-            logger.warning("Empty page content. Returning score 0.")
+           # logger.warning("Empty page content. Returning score 0.")
             return 0
         
         page_embedding = co.embed(texts=[page_content], model='embed-english-v3.0', input_type='search_document').embeddings[0]
         query_embedding = co.embed(texts=[query], model='embed-english-v3.0', input_type='search_query').embeddings[0]
         score = cosine_similarity([query_embedding], [page_embedding])[0][0]
-        logger.debug(f"Relevance score calculated: {score}")
+     #   logger.debug(f"Relevance score calculated: {score}")
         return score
     except Exception as e:
-        logger.exception(f"Error calculating relevance score: {str(e)}")
+       # logger.exception(f"Error calculating relevance score: {str(e)}")
         st.error(f"Error calculating relevance score: {str(e)}")
         return 0
 
 def analyze_competitors(row, co):
-    logger.info(f"Analyzing competitors for query: {row['query']}")
+   # logger.info(f"Analyzing competitors for query: {row['query']}")
     query = row['query']
     our_url = row['page']
     
@@ -184,14 +188,14 @@ def analyze_competitors(row, co):
     
     # Calculate score for our page first
     our_content = fetch_content(our_url)
-    print(out_url)
+    print(our_url)
     print(our_content)
     if our_content:
         our_score = calculate_relevance_score(our_content, query, co)
         results.append({'url': our_url, 'relevancy_score': our_score})
-        logger.info(f"Our URL: {our_url}, Score: {our_score}")
+        #logger.info(f"Our URL: {our_url}, Score: {our_score}")
     else:
-        logger.warning(f"No content fetched for our URL: {our_url}")
+        #logger.warning(f"No content fetched for our URL: {our_url}")
     
     # Calculate scores for competitor pages
     for url in competitor_urls:
@@ -199,25 +203,25 @@ def analyze_competitors(row, co):
            # logger.debug(f"Processing competitor URL: {url}")
             content = fetch_content(url)
             if not content:
-                logger.warning(f"No content fetched for competitor URL: {url}")
+        #        logger.warning(f"No content fetched for competitor URL: {url}")
                 continue
             
             score = calculate_relevance_score(content, query, co)
             
-            logger.info(f"Competitor URL: {url}, Score: {score}")
+          #  logger.info(f"Competitor URL: {url}, Score: {score}")
             results.append({'url': url, 'relevancy_score': score})
         except Exception as e:
-            logger.error(f"Error processing URL {url}: {str(e)}")
+          #  logger.error(f"Error processing URL {url}: {str(e)}")
             st.error(f"Error processing URL {url}: {str(e)}")
     
     results_df = pd.DataFrame(results).sort_values('relevancy_score', ascending=False)
     
-    logger.info(f"Competitor analysis completed. {len(results)} results obtained.")
+  #  logger.info(f"Competitor analysis completed. {len(results)} results obtained.")
     return results_df
 
 def show_competitor_analysis(row, co):
     if st.button("Check Competitors", key=f"comp_{row['page']}"):
-        logger.info(f"Competitor analysis requested for page: {row['page']}")
+       # logger.info(f"Competitor analysis requested for page: {row['page']}")
         with st.spinner('Analyzing competitors...'):
             results_df = analyze_competitors(row, co)
             st.write("Relevancy Score Comparison:")
@@ -226,7 +230,7 @@ def show_competitor_analysis(row, co):
             our_data = results_df[results_df['url'] == row['page']]
             if our_data.empty:
                 st.error(f"Our page '{row['page']}' is not in the results. This indicates an error in fetching or processing the page.")
-                logger.error(f"Our page '{row['page']}' is missing from the results.")
+               # logger.error(f"Our page '{row['page']}' is missing from the results.")
                 
                 # Additional debugging information
                 # st.write("Debugging Information:")
@@ -241,7 +245,7 @@ def show_competitor_analysis(row, co):
                 total_results = len(results_df)
                 our_score = our_data['relevancy_score'].values[0]
                 
-                logger.info(f"Our page ranks {our_rank} out of {total_results} in terms of relevancy score.")
+               # logger.info(f"Our page ranks {our_rank} out of {total_results} in terms of relevancy score.")
                 st.write(f"Our page ('{row['page']}') ranks {our_rank} out of {total_results} in terms of relevancy score.")
                 st.write(f"Our relevancy score: {our_score:.4f}")
                 
@@ -280,7 +284,7 @@ def analyze_competitors(row, co):
     
     return results_df
 def process_gsc_data(df):
-    logging.info("Processing GSC data")
+    #logging.info("Processing GSC data")
     df_sorted = df.sort_values(['impressions'], ascending=[False])
     df_unique = df_sorted.drop_duplicates(subset='page', keep='first')
     
@@ -290,7 +294,7 @@ def process_gsc_data(df):
         df_unique['relevancy_score'] = df_sorted.groupby('page')['relevancy_score'].first().values
     
     result = df_unique[['page', 'query', 'clicks', 'impressions', 'ctr', 'position', 'relevancy_score']]
-    logging.info("GSC data processed successfully")
+    #logging.info("GSC data processed successfully")
     return result
 
 # -------------
@@ -298,7 +302,7 @@ def process_gsc_data(df):
 # -------------
 
 def load_config():
-    logging.info("Loading Google client configuration")
+    #logging.info("Loading Google client configuration")
     client_config = {
         "web": {
             "client_id": os.environ["CLIENT_ID"],
@@ -308,25 +312,25 @@ def load_config():
             "redirect_uris": ["https://poemsforaphrodite-gscpro.hf.space/"],
         }
     }
-    logging.info("Google client configuration loaded")
+    #logging.info("Google client configuration loaded")
     return client_config
 
 def init_oauth_flow(client_config):
-    logging.info("Initializing OAuth flow")
+    #logging.info("Initializing OAuth flow")
     scopes = ["https://www.googleapis.com/auth/webmasters.readonly"]
     flow = Flow.from_client_config(
         client_config,
         scopes=scopes,
         redirect_uri=client_config["web"]["redirect_uris"][0]
     )
-    logging.info("OAuth flow initialized")
+    #logging.info("OAuth flow initialized")
     return flow
 
 def google_auth(client_config):
-    logging.info("Starting Google authentication")
+   # logging.info("Starting Google authentication")
     flow = init_oauth_flow(client_config)
     auth_url, _ = flow.authorization_url(prompt="consent")
-    logging.info("Google authentication URL generated")
+    #logging.info("Google authentication URL generated")
     return flow, auth_url
 
 def auth_search_console(client_config, credentials):
@@ -348,7 +352,7 @@ def auth_search_console(client_config, credentials):
 # -------------
 
 def list_gsc_properties(credentials):
-    logging.info("Listing GSC properties")
+   # logging.info("Listing GSC properties")
     service = build('webmasters', 'v3', credentials=credentials)
     site_list = service.sites().list().execute()
     properties = [site['siteUrl'] for site in site_list.get('siteEntry', [])] or ["No properties found"]
@@ -362,16 +366,16 @@ def fetch_gsc_data(webproperty, search_type, start_date, end_date, dimensions, d
         query = query.filter('device', 'equals', device_type.lower())
     try:
         df = query.limit(MAX_ROWS).get().to_dataframe()
-        logging.info("GSC data fetched successfully")
+        #logging.info("GSC data fetched successfully")
         return process_gsc_data(df)
     except Exception as e:
-        logging.error(f"Error fetching GSC data: {e}")
+        #logging.error(f"Error fetching GSC data: {e}")
         show_error(e)
         return pd.DataFrame()
 
     
 def calculate_relevancy_scores(df, model_type):
-    logging.info("Calculating relevancy scores")
+    #logging.info("Calculating relevancy scores")
     with st.spinner('Calculating relevancy scores...'):
         try:
             page_contents = [fetch_content(url) for url in df['page']]
@@ -379,9 +383,9 @@ def calculate_relevancy_scores(df, model_type):
             query_embeddings = generate_embeddings(df['query'].tolist(), model_type)
             relevancy_scores = cosine_similarity(query_embeddings, page_embeddings).diagonal()
             df = df.assign(relevancy_score=relevancy_scores)
-            logging.info("Relevancy scores calculated successfully")
+            #logging.info("Relevancy scores calculated successfully")
         except Exception as e:
-            logging.error(f"Error calculating relevancy scores: {e}")
+            #logging.error(f"Error calculating relevancy scores: {e}")
             st.warning(f"Error calculating relevancy scores: {e}")
             df = df.assign(relevancy_score=0)
     return df
@@ -417,7 +421,7 @@ def calc_date_range(selection, custom_start=None, custom_end=None):
     return date_range
 
 def show_error(e):
-    logging.error(f"An error occurred: {e}")
+    #logging.error(f"An error occurred: {e}")
     st.error(f"An error occurred: {e}")
 
 def property_change():
@@ -429,33 +433,33 @@ def property_change():
 # -------------
 
 def show_dataframe(report):
-    logging.info("Showing dataframe preview")
+    #logging.info("Showing dataframe preview")
     with st.expander("Preview the First 100 Rows (Unique Pages with Top Query)"):
         st.dataframe(report.head(DF_PREVIEW_ROWS))
 
 def download_csv_link(report):
-    logging.info("Generating CSV download link")
+    #logging.info("Generating CSV download link")
     def to_csv(df):
         return df.to_csv(index=False, encoding='utf-8-sig')
     csv = to_csv(report)
     b64_csv = base64.b64encode(csv.encode()).decode()
     href = f'<a href="data:file/csv;base64,{b64_csv}" download="search_console_data.csv">Download CSV File</a>'
     st.markdown(href, unsafe_allow_html=True)
-    logging.info("CSV download link generated")
+    #logging.info("CSV download link generated")
 
 # -------------
 # Streamlit UI Components
 # -------------
 
 def show_google_sign_in(auth_url):
-    logging.info("Showing Google sign-in button")
+   # logging.info("Showing Google sign-in button")
     with st.sidebar:
         if st.button("Sign in with Google"):
             st.write('Please click the link below to sign in:')
             st.markdown(f'[Google Sign-In]({auth_url})', unsafe_allow_html=True)
 
 def show_property_selector(properties, account):
-    logging.info("Showing property selector")
+  #  logging.info("Showing property selector")
     selected_property = st.selectbox(
         "Select a Search Console Property:",
         properties,
@@ -467,7 +471,7 @@ def show_property_selector(properties, account):
     return account[selected_property]
 
 def show_search_type_selector():
-    logging.info("Showing search type selector")
+  #  logging.info("Showing search type selector")
     return st.selectbox(
         "Select Search Type:",
         SEARCH_TYPES,
@@ -476,7 +480,7 @@ def show_search_type_selector():
     )
 
 def show_model_type_selector():
-    logging.info("Showing model type selector")
+  #  logging.info("Showing model type selector")
     return st.selectbox(
         "Select the embedding model:",
         ["english", "multilingual"],
@@ -512,7 +516,7 @@ def show_tabular_data(df, co):
                         
 
 def show_date_range_selector():
-    logging.info("Showing date range selector")
+  #  logging.info("Showing date range selector")
     return st.selectbox(
         "Select Date Range:",
         DATE_RANGE_OPTIONS,
@@ -521,12 +525,12 @@ def show_date_range_selector():
     )
 
 def show_custom_date_inputs():
-    logging.info("Showing custom date inputs")
+   # logging.info("Showing custom date inputs")
     st.session_state.custom_start_date = st.date_input("Start Date", st.session_state.custom_start_date)
     st.session_state.custom_end_date = st.date_input("End Date", st.session_state.custom_end_date)
 
 def show_dimensions_selector(search_type):
-    logging.info("Showing dimensions selector")
+  #  logging.info("Showing dimensions selector")
     available_dimensions = update_dimensions(search_type)
     return st.multiselect(
         "Select Dimensions:",
@@ -536,7 +540,7 @@ def show_dimensions_selector(search_type):
     )
 
 def show_paginated_dataframe(report, rows_per_page=20):
-    logging.info("Showing paginated dataframe")
+  #  logging.info("Showing paginated dataframe")
     report['position'] = report['position'].astype(int)
     report['impressions'] = pd.to_numeric(report['impressions'], errors='coerce')
     
@@ -608,7 +612,7 @@ def show_paginated_dataframe(report, rows_per_page=20):
 # -------------
 
 def main():
-    logging.info("Starting main function")
+   # logging.info("Starting main function")
     setup_streamlit()
     client_config = load_config()
     
@@ -653,17 +657,17 @@ def main():
                 st.write("Data fetched successfully. Click the button below to calculate relevancy scores.")
                 
                 if st.button("Calculate Relevancy Scores"):
-                    logger.info("Calculating relevancy scores for all rows")
+                  #  logger.info("Calculating relevancy scores for all rows")
                     st.session_state.report_data = calculate_relevancy_scores(st.session_state.report_data, model_type)
                 
                 show_tabular_data(st.session_state.report_data, co)
                 
                 download_csv_link(st.session_state.report_data)
             elif st.session_state.report_data is not None:
-                logger.warning("No data found for the selected criteria.")
+               # logger.warning("No data found for the selected criteria.")
                 st.warning("No data found for the selected criteria.")
 
 if __name__ == "__main__":
-    logging.info("Running main function")
+   # logging.info("Running main function")
     main()
-    logger.info("Script completed")
+    #logger.info("Script completed")