|
from selenium import webdriver |
|
from selenium.webdriver.common.by import By |
|
import undetected_chromedriver as uc |
|
import re |
|
import logging |
|
import os |
|
import time |
|
import random |
|
from config import SCRAPER_TIMEOUT, CHROME_DRIVER_PATH, SCRAPER_MAX_RETRIES |
|
|
|
|
|
def get_text(url, n_words=15): |
|
try: |
|
driver = None |
|
logging.warning(f"Initiated Scraping {url}") |
|
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/123.0.0.0 Safari/537.36" |
|
options = uc.ChromeOptions() |
|
options.add_argument("--headless") |
|
options.add_argument(f"user-agent={user_agent}") |
|
options.add_argument("--blink-settings=imagesEnabled=false") |
|
options.add_argument("--disable-images") |
|
options.add_argument("--disable-blink-features=AutomationControlled") |
|
options.add_argument("--disable-dev-shm-usage") |
|
|
|
|
|
|
|
|
|
|
|
|
|
driver = uc.Chrome(version_main=127, options=options, driver_executable_path=CHROME_DRIVER_PATH) |
|
time.sleep(random.uniform(0.5, 1.5)) |
|
driver.set_page_load_timeout(SCRAPER_TIMEOUT) |
|
driver.set_script_timeout(SCRAPER_TIMEOUT) |
|
driver.implicitly_wait(3) |
|
driver.get(url) |
|
elem = driver.find_element(By.TAG_NAME, "body").text |
|
sents = elem.split("\n") |
|
sentence_list = [] |
|
for sent in sents: |
|
sent = sent.strip() |
|
if (len(sent.split()) >= n_words) and (len(re.findall(r"^\w.+[^\w\)\s]$", sent))>0): |
|
sentence_list.append(sent) |
|
driver.close() |
|
driver.quit() |
|
logging.warning("Closed Webdriver") |
|
logging.warning("Successfully scraped text") |
|
if len(sentence_list) < 3: |
|
raise Exception("Found nothing to scrape.") |
|
return "\n".join(sentence_list), "" |
|
except Exception as e: |
|
logging.warning(str(e)) |
|
if driver: |
|
driver.close() |
|
driver.quit() |
|
logging.warning("Closed Webdriver") |
|
err_msg = str(e).split('\n')[0] |
|
return "", err_msg |
|
|
|
|
|
def scrape_text(url, n_words=15,max_retries=SCRAPER_MAX_RETRIES): |
|
scraped_text = "" |
|
scrape_error = "" |
|
try: |
|
n_tries = 1 |
|
while (n_tries <= max_retries) and (scraped_text == ""): |
|
scraped_text, scrape_error = get_text(url=url, n_words=n_words) |
|
n_tries += 1 |
|
return scraped_text, scrape_error |
|
except Exception as e: |
|
err_msg = str(e).split('\n')[0] |
|
return "", err_msg |
|
|
