Update trend_crawl.py

trend_crawl.py

@@ -31,17 +31,18 @@ def setup_driver():
 
 def process_selenium_row(index, selenium_rows, driver):
     """Extract dynamic data using Selenium by clicking on the row."""
-    max_retries = 3
+    max_retries = 5  # Increase retries
     for attempt in range(max_retries):
         try:
+            selenium_rows = driver.find_elements(By.CSS_SELECTOR, '[jsname="oKdM2c"]')  # Refresh rows
             row = selenium_rows[index]
-            row.click()
-
-            # Wait for elements with class="xZCHj" to load
-            WebDriverWait(driver, 10).until(
+            driver.execute_script("arguments[0].click();", row)  # Use JavaScript click
+            time.sleep(1)  # Adjust delay
+            
+            # Wait for dynamic content
+            WebDriverWait(driver, 15).until(
                 EC.presence_of_all_elements_located((By.CLASS_NAME, "xZCHj"))
             )
-
             links = driver.find_elements(By.CLASS_NAME, "xZCHj")
             dynamic_data = {
                 "article": [
@@ -52,12 +53,11 @@ def process_selenium_row(index, selenium_rows, driver):
                     for link in links
                 ]
             }
-
             if dynamic_data["article"]:
                 return dynamic_data
         except Exception as e:
             print(f"Error processing row {index} (Attempt {attempt + 1}): {e}")
-            selenium_rows = driver.find_elements(By.CSS_SELECTOR, '[jsname="oKdM2c"]')
+            time.sleep(2)  # Add delay before retry
 
     print(f"Failed to process row {index} after {max_retries} attempts.")
     return {"article": []}