ksvmuralidhar commited on
Commit
512cc5f
·
verified ·
1 Parent(s): 1586ad3

Update scraper.py

Browse files
Files changed (1) hide show
  1. scraper.py +2 -2
scraper.py CHANGED
@@ -6,7 +6,7 @@ import logging
6
  import os
7
  import time
8
  import random
9
- from config import SCRAPER_TIMEOUT, CHROME_DRIVER_PATH
10
 
11
 
12
  def get_text(url, n_words=15):
@@ -57,7 +57,7 @@ def get_text(url, n_words=15):
57
  return "", err_msg
58
 
59
 
60
- def scrape_text(url, n_words=15,max_retries=2):
61
  scraped_text = ""
62
  scrape_error = ""
63
  try:
 
6
  import os
7
  import time
8
  import random
9
+ from config import SCRAPER_TIMEOUT, CHROME_DRIVER_PATH, SCRAPER_MAX_RETRIES
10
 
11
 
12
  def get_text(url, n_words=15):
 
57
  return "", err_msg
58
 
59
 
60
+ def scrape_text(url, n_words=15,max_retries=SCRAPER_MAX_RETRIES):
61
  scraped_text = ""
62
  scrape_error = ""
63
  try: