Update news_extractor/news_extractor.py

news_extractor/news_extractor.py

@@ -6,6 +6,7 @@ import regex as re
 from dateutil import parser
 import logging
 import multiprocessing
+from config import NEWS_EXTRACTOR_URL_TIMEOUT, RSS_FEEDS_TO_EXTRACT
 
 
 def date_time_parser(dt):
@@ -91,9 +92,8 @@ def news_agg(rss):
                     'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/115.0.0.0 Safari/537.36'
         }
 
-        timeout = 5
         
-        resp = r.get(rss, timeout=timeout, headers=headers)
+        resp = r.get(rss, timeout=NEWS_EXTRACTOR_URL_TIMEOUT, headers=headers)
         logging.warning(f'{rss}: {resp.status_code}')
         b = BeautifulSoup(resp.content, "xml")
         items = b.find_all("item")
@@ -121,19 +121,7 @@ def news_agg(rss):
 
 
 # List of RSS feeds
-rss = ['https://www.economictimes.indiatimes.com/rssfeedstopstories.cms',
-       'https://www.thehindu.com/news/feeder/default.rss',
-       # 'https://telanganatoday.com/feed',
-       'https://www.businesstoday.in/rssfeeds/?id=225346',
-       'https://feeds.feedburner.com/ndtvnews-latest',
-       'https://www.hindustantimes.com/feeds/rss/world-news/rssfeed.xml',
-       'https://www.indiatoday.in/rss/1206578',
-       
-       'https://www.moneycontrol.com/rss/latestnews.xml',
-       'https://www.livemint.com/rss/news',
-       
-       'https://www.zeebiz.com/latest.xml/feed',
-       'https://www.timesofindia.indiatimes.com/rssfeedmostrecent.cms']
+rss = RSS_FEEDS_TO_EXTRACT
 
 
 def get_news_rss(url):