web_scrapping_engine.py

import os
from linkedin import get_linkedin_profile
from company_website import full_company_website_exploration

def run_web_scrapping_engine(company_name = "sunday natural", folder_path = ".", output_in_code = False):
    path = "."
    os.makedirs(path, exist_ok=True)
    os.makedirs(f"{path}/pdf/", exist_ok=True)
    print("Starting Linkedin gathering...")
    linkedin = get_linkedin_profile(company_name, folder_path=path)
    print("Linkedin Done!\n========> Starting now company website gathering...")
    content = full_company_website_exploration(company_name, folder_path=path)
    content.append(linkedin)
    if output_in_code:
        return content