obsei_module/obsei/misc/web_search.py

from typing import Any, Dict, List, Optional

import requests
from requests.adapters import HTTPAdapter
from urllib3 import Retry

GOOGLE_SEARCH_URL = "https://www.google.com/search"


# Code is influenced from https://github.com/cowboy-bebug/app-store-scraper
def perform_search(
    request_url: str,
    query: str,
    search_url: str = GOOGLE_SEARCH_URL,
    search_country: Optional[str] = None,
    headers: Optional[Dict[str, Any]] = None,
    total: int = 3,
    backoff_factor: int = 3,
    status_force_list: Optional[List[int]] = None,
) -> requests.Response:

    params = {"q": query}
    if search_country:
        params["cr"] = search_country

    if not status_force_list:
        status_force_list = [404, 429]
    retries = Retry(
        total=total,
        backoff_factor=backoff_factor,
        status_forcelist=status_force_list,
    )
    with requests.Session() as s:
        s.mount(request_url, HTTPAdapter(max_retries=retries))
        return s.get(search_url, headers=headers, params=params)