Upload 19 files

Daily_Star_fully_scraped.py

@@ -0,0 +1,79 @@
+def get_data(number):
+
+    
+    ##Necessary imports
+    from selenium import webdriver
+    from selenium.webdriver import chrome
+    from selenium.webdriver import ChromeOptions
+    options = ChromeOptions()
+    options.add_argument("--headless=new")
+    driver = webdriver.Chrome(options=options)
+    ## Finding Elements by XPATH
+    from selenium.webdriver.common.by import By
+
+    driver.get("https://www.thedailystar.net/tags/road-accident")
+    
+    """
+    Browsing with browser open codes:
+    ##Necessary imports
+    from selenium import webdriver
+    from selenium.webdriver import chrome
+
+    driver = webdriver.Chrome()
+    ## Finding Elements by XPATH
+    from selenium.webdriver.common.by import By
+    driver.get("https://en.prothomalo.com/topic/Road-accident")
+   """ 
+    import time
+    news_list=[]
+    news_link=[]
+    publish_date=[]
+    for i in range(number):
+        #time.sleep(5)
+        if (i+1)!=0 and (i+1)%10==0:
+            last_height = driver.execute_script("return document.body.scrollHeight")
+            driver.execute_script(f"window.scrollTo(0, {last_height-950})")
+            driver.find_element('xpath',f'/html/body/div[3]/div/div/div/div[2]/main/div/div[2]/div/div[5]/div/div/div[1]/div/div[2]/ul/li/a').click()
+            time.sleep(10)
+        txt=driver.find_element('xpath',f'/html/body/div[3]/div/div/div/div[2]/main/div/div[2]/div/div[5]/div/div/div[1]/div/div[1]/div[{i+1}]/div[2]/div[2]/div[2]/h3/a')
+        publish_date.append(driver.find_element('xpath',f'/html/body/div[3]/div/div/div/div[2]/main/div/div[2]/div/div[5]/div/div/div[1]/div/div[1]/div[{i+1}]/div[2]/div[1]').text)
+        news_list.append(txt.text)
+        news_link.append(txt.get_attribute("href"))   
+
+
+
+    #### Converting the list to a pandas dataframe by converting the list to a dictionary  ###
+    dict={'News Title':news_list,'News Link':news_link,'Publish Date':publish_date}
+    import pandas as pd
+    df=pd.DataFrame(dict)
+
+
+    ############################################### Description Exctraction #################################################
+    from newspaper import Article
+
+
+    text=[]
+    for i in range(len(df)):
+        url = df['News Link'][i]
+        article = Article(url)
+        article.download()
+        article.parse()
+        
+        text.append(article.text)
+
+    df2=df.assign(Description=text)
+
+
+    for p in range(len(df2)):
+        if df2['Publish Date'][p]=="Not available":
+            df2.drop([p],inplace=True)
+    #df2.reset_index()
+
+    df2.reset_index(drop=True,inplace=True)
+
+    df2["Date + Desc"]=df2['Publish Date'] + ".     News Description:"+ df2['Description']
+
+
+    
+    return df2
+    #df3.to_csv('Prothom_Alo_Description.txt',  index=False)

Dhaka_Tribune_Fully_Scraped.py

@@ -0,0 +1,82 @@
+def get_data(number):
+    ##Necessary imports
+    from selenium import webdriver
+    from selenium.webdriver import chrome
+    from selenium.webdriver import ChromeOptions
+    import math
+    options = ChromeOptions()
+    options.add_argument("--headless=new")
+    driver = webdriver.Chrome(options=options)
+    ## Finding Elements by XPATH
+    from selenium.webdriver.common.by import By
+
+
+    driver.get("https://www.dhakatribune.com/topic/road-accident")
+
+    #### Scraping News Title and News Link ####
+    import time
+    news_list=[]
+    news_link=[]
+    publish_date=[]
+    row_counter=0
+    news_counter=0
+    for i in range(number):
+        if i==0:
+            row_counter=1
+        else:
+            row_counter=math.ceil(i/4)
+        news_counter=i%4+1
+        #time.sleep(5)
+        if (i+1)!=0 and (i+1)%20==0:
+            last_height = driver.execute_script("return document.body.scrollHeight")
+            driver.execute_script(f"window.scrollTo(0, {last_height})")
+            driver.find_element('xpath',f'/html/body/div[3]/div/div[2]/div/div/div[2]/div/div/div/div[2]/div/div[2]/button').click()
+            time.sleep(10)
+        txt=driver.find_element('xpath',f'/html/body/div[3]/div/div[2]/div/div/div[2]/div/div/div/div[2]/div/div[1]/div[{row_counter}]/div[{news_counter}]/div/div[2]/div/div/div/h2/a')
+        #publish_date.append(driver.find_element('xpath',f'/html/body/div[3]/div/div/div/div[2]/main/div/div[2]/div/div[5]/div/div/div[1]/div/div[1]/div[{i+1}]/div[1]').text)
+        news_list.append(txt.text)
+        news_link.append(txt.get_attribute("href")) 
+    
+    ###### Scraping Publish Date ######
+    publish_date=[]
+    for i in range (len(news_link)):
+        driver.get(news_link[i])
+        time.sleep(6)
+        driver.execute_script("window.stop();")
+        try:
+            publish_date.append(driver.find_element('xpath','/html/body/div[3]/div/div[2]/div/div/div[2]/div/div[1]/div/div[1]/div/div/div/div/div/div/div[2]/div/div/div[3]/div/div[1]/div/div[2]/span[1]').text)
+        except:
+            publish_date.append("Not available")
+
+    #### Converting the list to a pandas dataframe by converting the list to a dictionary  ###
+    dict={'News Title':news_list,'News Link':news_link,'Publish Date':publish_date}
+    import pandas as pd
+    df=pd.DataFrame(dict)
+
+
+    ############################################ Description Extraction ###################################################
+
+    from newspaper import Article
+    text=[]
+    for i in range(len(df)):
+        url = df['News Link'][i]
+        article = Article(url)
+        article.download()
+        article.parse()
+        
+        text.append(article.text)
+
+
+    df2=df.assign(Description=text)
+    for p in range(len(df2)):
+        if df2['Publish Date'][p]=="Not available":
+            df2.drop([p],inplace=True)
+
+    df2.reset_index(drop=True,inplace=True)
+    df2["Date + Desc"]=df2['Publish Date'] + ".     News Description:"+ df2['Description']
+
+
+   
+    return df2
+    
+    #df3.to_csv('Dhaka_Tribune_Description.txt',  index=False)

LLM_automation_GPT.py

@@ -0,0 +1,125 @@
+def create_data(description):
+    print("Running THis Script")
+    from langchain_core.prompts import ChatPromptTemplate  ### To create a chatbot, chatprompttemplate used
+    from langchain_openai import ChatOpenAI ##### For using chat openai features
+    from langchain_core.output_parsers import StrOutputParser  ### Default output parser. Custom parser can also be created
+    from langchain_community.llms import ollama ### Importing ollama
+
+    
+    import os
+    from dotenv import load_dotenv
+    import pandas as pd
+
+
+    load_dotenv()
+
+    ### Set all api keys:
+    os.environ["OPENAI_API_KEY"]="sk-proj-ZB9b6Gn2FccVRsaL9WYfT3BlbkFJDpUpcoUwyR9LPoIJuAVl"
+    ### Create Prompt Template:
+    prompt=ChatPromptTemplate.from_messages(
+        {
+            ("system", "You are a helpful assistant, please respond to the queries"), ### We need both system and users in prompt
+            ("user","question: {question}")
+        }
+    )
+
+    #### Create OpenAI llm:
+    llm=ChatOpenAI(model="gpt-4o")
+
+    ### Create an output parser:
+    output_parser=StrOutputParser()
+
+    #### Creating chain: The concept is- output of action before | symbol will be passed as input in action after the symbol.
+    #### Here we have created three actions: The prompt, llm and output parser:
+    chain=prompt|llm|output_parser
+
+    df = description
+    df = df.fillna(0)
+    dj=[]
+
+    for i in range(len(df)):
+        dj.append(chain.invoke({"question" : df['Date + Desc'][i]+" Is the news referring to one or many specific accident incidents or accident in general? Make sure that your answer is only in one word. If a report contains more than one accident incident, classify it as a general accident incident. The word should be either 'Specific' or 'General'. Your answer should not contain any words except 'Specific' and 'General'  "}))
+        
+    df2=df.copy()
+    df2['Report Type']=dj
+    def drp(p):
+        df2.drop([p],inplace=True)
+
+    ### Removing the general accident types:
+    for p in range(len(df)):
+        if "General" in df2['Report Type'][p]:
+            drp(p)
+            
+    ### Reseting index of df3:
+    df2.reset_index(drop=True,inplace=True)
+
+    ### Now finding column values using llm:
+    ### A function to invoke the llm. For some reason phi3 doesn't give accurate result sometimes if used directly in dj.append()
+    def res(i):
+        response=chain.invoke({"question" : df2['Description'][i]+f"""Provide only the answers of the following question seperated by a comma only:
+                            If the news was published on {df2['Publish Date'][i]}, what is the date of accident occurrence? The date must be in Day-Month-Year format. Be careful because publish date and accident occurrence date may or may not be the same. Try to deduce correct accident date and do not include Saturday Sunday etc in your date. Only numerics are needed,
+                            Time of Accident occured, How many people were killed in the accident in numeric number?, 
+                            How many people were injured in the accident in numeric number?, 
+                            Location of the accident, 
+                            Type of road where accident occured, 
+                            Was there any pedestrian involved?,  
+                            Do not include any other sentences except the answers seperated by comma only, 
+                            if you cannot find or deduce a answer simply put 'Not Available' in place of it. 
+                            If a report mentions more than one specific accident incidents only consider the 1st accident incident and ignore the second one""" })
+        return response
+    #### dj2 list contains all column values seperated by comma:
+    dj2=[]
+
+    for i in range(len(df2)):
+        dj2.append(res(i))
+    ### Finding vehicle
+    def res2(i):
+        response=chain.invoke({"question" : df2['Date + Desc'][i]+" Only name the type of vehicles involved in the accident. If multiple vehicles are involved, seperate them by hyphens(-). Example answers: Bus, Truck-Bus etc. If no vehicles are mentioned, your answer will be: Not Available. Your answer should only contain the vehicle name, do not include any extra sentences"})
+        return response
+    #### vehicle list contains all vehicles involved:
+    vehicles=[]
+
+    for i in range(len(df2)):
+        vehicles.append(res2(i))
+   
+
+
+
+    ### Splitting dj2 string based on comma position:
+    Date=[]
+    Time=[]
+    Killed=[]
+    Injured=[]
+    Location=[]
+    Road_Characteristic=[]
+    Pedestrian_Involved=[]
+    #Vehicles_involved=[]
+
+    for i in range(len(dj2)):
+        words = dj2[i].split(",")  # Splitting at the comma delimiter
+        #print(f"Date: {words[0]}")
+        Date.append(words[0])
+            
+        #print(f"Time: {words[1]}")
+        Time.append(words[1])
+            
+        #print(f"Casualities: {words[2]}")
+        Killed.append(words[2])
+        Injured.append(words[3])
+        Location.append(words[4])
+        Road_Characteristic.append(words[5])
+        Pedestrian_Involved.append(words[6])
+        #Vehicles_involved.append(words[7])
+
+    #### Probable type of final dataframe:
+    df2["Accident Date"]=Date
+    df2["Time"]=Time
+    df2["Killed"]=Killed
+    df2["Injured"]=Injured
+    df2["Location"]=Location
+    df2["Road_Characteristic"]=Road_Characteristic
+    df2["Pedestrian_Involved"]=Pedestrian_Involved
+    df2["Vehicles Involved"]=vehicles
+    df3=df2.drop(columns=['Description','Date + Desc','Report Type'])
+    return df3
+   

LLM_automation_GPT35.py

@@ -0,0 +1,91 @@
+def create_data(description):
+    from langchain_core.prompts import ChatPromptTemplate  ### To create a chatbot, chatprompttemplate used
+    from langchain_openai import ChatOpenAI ##### For using chat openai features
+    from langchain_core.output_parsers import StrOutputParser  ### Default output parser. Custom parser can also be created
+    
+
+
+    import os
+    from dotenv import load_dotenv
+
+
+    load_dotenv()
+
+    ### Set all api keys:
+    os.environ["OPENAI_API_KEY"]="sk-proj-ZB9b6Gn2FccVRsaL9WYfT3BlbkFJDpUpcoUwyR9LPoIJuAVl"
+
+
+    ### Create Prompt Template:
+    prompt=ChatPromptTemplate.from_messages(
+        {
+            ("system", "You are a helpful assistant, please respond to the queries"), ### We need both system and users in prompt
+            ("user","question: {question}")
+        }
+    )
+    df2=description
+    #### Create OpenAI llm:
+    llm=ChatOpenAI(model="gpt-3.5-turbo")
+
+    ### Create an output parser:
+    output_parser=StrOutputParser()
+
+    #### Creating chain: The concept is- output of action before | symbol will be passed as input in action after the symbol.
+    #### Here we have created three actions: The prompt, llm and output parser:
+    chain=prompt|llm|output_parser
+
+    ### A function to invoke the llm. For some reason phi3 doesn't give accurate result sometimes if used directly in dj.append()
+    def res(i):
+        response=chain.invoke({"question" : df2['Description'][i]+" Is the news referring to a specific accident incident or accident in general? Answer only in a word: 'specific' or 'general'. No other words are allowed in your answer"})
+        return response
+
+    #### dj list contains type of report 'General' or 'Specific'
+    dj=[]
+
+    for i in range(len(df2)):
+        dj.append(res(i))
+
+    df2['Report Type']=dj
+
+    def drp(p):
+        df2.drop([p],inplace=True)
+    ### Removing the general accident types:
+    for p in range(len(df2)):
+        if "General" in df2['Report Type'][p] or "general" in df2['Report Type'][p]:
+            drp(p)
+            
+    ### Reseting index of df3:
+    df2.reset_index(drop=True,inplace=True)
+
+
+    ### Splitting dj2 string based on comma position:
+    Date=[]
+    Time=[]
+    Killed=[]
+    Injured=[]
+    Location=[]
+    Road_Characteristic=[]
+    Pedestrian_Involved=[]
+    vehicles=[]
+    #Weather=[]
+
+    for i in range(len(df2)):
+        Date.append(chain.invoke({"question" : "Read the accident report carefully and provide only the answer of the question asked. Do not add any extra sentences or words except the answer: What is the date of accident occurrence in Day-Month-Year format. Keep in mind that news publish date and accident occurrence date may be different. If you cannot find or deduce the answer, simply reply Not Available" + df2['Description'][i]}))
+        Time.append(chain.invoke({"question" : "Read the accident report carefully and provide only the answer of the question asked. Do not add any extra sentences or words except the answer: What is the time of accident occurrence in 24-hour format. If you cannot find or deduce the answer, simply reply Not Available" + df2['Description'][i]}))
+        Killed.append(chain.invoke({"question" : "Read the accident report carefully and provide only the answer of the question asked. Do not add any extra sentences or words except the answer: How many people were killed in the accident?. If you cannot find or deduce the answer, simply reply Not Available" + df2['Description'][i]}))
+        Injured.append(chain.invoke({"question" : "Read the accident report carefully and provide only the answer of the question asked. Do not add any extra sentences or words except the answer: How many people were injured in the accident?. If you cannot find or deduce the answer, simply reply Not Available" + df2['Description'][i]}))
+        Location.append(chain.invoke({"question" : "Read the accident report carefully and provide only the answer of the question asked. Do not add any extra sentences or words except the answer: What is the name of the location where accident took place?. If you cannot find or deduce the answer, simply reply Not Available" + df2['Description'][i]}))
+        Road_Characteristic.append(chain.invoke({"question" : "Read the accident report carefully and provide only the answer of the question asked. Do not add any extra sentences or words except the answer: What is the type of road where accident took place?. If you cannot find or deduce the answer, simply reply Not Available" + df2['Description'][i]}))
+        Pedestrian_Involved.append(chain.invoke({"question" : "Read the accident report carefully and provide only the answer of the question asked. Do not add any extra sentences or words except the answer: Was there any pedestrian involved in the accident?. If you cannot find or deduce the answer, simply reply Not Available" + df2['Description'][i]}))
+        vehicles.append(chain.invoke({"question" : "Only name the type of vehicles involved in the accident. If multiple vehicles are involved, seperate them by hyphens(-). Example answers: Bus, Truck-Bus etc. If no vehicles are mentioned, your answer will be: Not Available. Your answer should only contain the vehicle name, do not include any extra sentences" + df2['Description'][i]}))
+        
+    #### Probable type of final dataframe:
+    df2["Date"]=Date
+    df2["Time"]=Time
+    df2["Killed"]=Killed
+    df2["Injured"]=Injured
+    df2["Location"]=Location
+    df2["Road_Characteristic"]=Road_Characteristic
+    df2["Pedestrian_Involved"]=Pedestrian_Involved
+    df2["Vehicles Involved"]=vehicles
+    df3=df2.drop(columns=['Description','Report Type','Date + Desc'])
+    return df3

LLM_automation_Groq.py

@@ -0,0 +1,130 @@
+def create_data(description):
+    from langchain_core.prompts import ChatPromptTemplate  ### To create a chatbot, chatprompttemplate used
+
+    from langchain_core.output_parsers import StrOutputParser  ### Default output parser. Custom parser can also be created
+    from langchain_groq import ChatGroq
+
+
+    import os
+    from dotenv import load_dotenv
+    import pandas as pd
+
+    load_dotenv()
+
+    ### Set all api keys:
+
+    #os.environ["LANGCHAIN_TRACING_V2"]="true" ### Will automatically trace our codes using Langsmith
+    os.environ["GROQ_API_KEY"]="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"  #### Will be used for monitoring the calls to and from llm (both free and paid)
+
+    ### Create Prompt Template:
+    prompt=ChatPromptTemplate.from_messages(
+        {
+            ("system", "You are a helpful assistant, please respond to the queries"), ### We need both system and users in prompt
+            ("user","question: {question}")
+        }
+    )
+
+    #### Create LLama3 70B llm:
+    llm = ChatGroq(
+        model="llama3-70b-8192"
+    )  # assuming you have Ollama installed and have llama3 model pulled with `ollama pull llama3 `
+
+
+    ### Create an output parser:
+    output_parser=StrOutputParser()
+
+    #### Creating chain: The concept is- output of action before | symbol will be passed as input in action after the symbol.
+    #### Here we have created three actions: The prompt, llm and output parser:
+    chain=prompt|llm|output_parser
+
+    df = description
+    df = df.fillna(0)
+    dj=[]
+
+    for i in range(len(df)):
+        dj.append(chain.invoke({"question" : df['Date + Desc'][i]+" Is the news referring to one or many specific accident incidents or accident in general? Make sure that your answer is only in one word. If a report contains more than one accident incident, classify it as a general accident incident. The word should be either 'Specific' or 'General'. Your answer should not contain any words except 'Specific' and 'General'  "}))
+        
+    df2=df.copy()
+    df2['Report Type']=dj
+    def drp(p):
+        df2.drop([p],inplace=True)
+
+    ### Removing the general accident types:
+    for p in range(len(df)):
+        if "General" in df2['Report Type'][p]:
+            drp(p)
+            
+    ### Reseting index of df3:
+    df2.reset_index(drop=True,inplace=True)
+
+    ### Now finding column values using llm:
+    ### A function to invoke the llm. For some reason phi3 doesn't give accurate result sometimes if used directly in dj.append()
+    def res(i):
+        response=chain.invoke({"question" : df2['Description'][i]+f"""Provide only the answers of the following question seperated by a comma only:
+                            If the news was published on {df2['Publish Date'][i]}, what is the date of accident occurrence? The date must be in Day-Month-Year format. Be careful because publish date and accident occurrence date may or may not be the same. Try to deduce correct accident date,
+                            Time of Accident occured, How many people were killed in the accident in numeric number?, 
+                            How many people were injured in the accident in numeric number?, 
+                            Location of the accident, 
+                            Type of road where accident occured, 
+                            Was there any pedestrian involved?,  
+                            Do not include any other sentences except the answers seperated by comma only and do not include sentences such as: Here are the answers, 
+                            if you cannot find or deduce a answer simply put 'Not Available' in place of it. 
+                            If a report mentions more than one specific accident incidents only consider the 1st accident incident and ignore the second one""" })
+        return response
+    #### dj2 list contains all column values seperated by comma:
+    dj2=[]
+
+    for i in range(len(df2)):
+        dj2.append(res(i))
+
+### A function to invoke the llm. For some reason phi3 doesn't give accurate result sometimes if used directly in dj.append()
+    def res2(i):
+        response=chain.invoke({"question" : df2['Date + Desc'][i]+" Only name the type of vehicles involved in the accident. If multiple vehicles are involved, seperate them by hyphens(-). Example answers: Bus, Truck-Bus etc. If no vehicles are mentioned, your answer will be: Not Available. Your answer should only contain the vehicle name, do not include any extra sentences"})
+        return response
+    #### dj2 list contains all column values seperated by comma:
+    vehicles=[]
+
+    for i in range(len(df2)):
+        vehicles.append(res2(i))
+
+
+    ### Splitting dj2 string based on comma position:
+    Date=[]
+    Time=[]
+    Killed=[]
+    Injured=[]
+    Location=[]
+    Road_Characteristic=[]
+    Pedestrian_Involved=[]
+    #Vehicles_involved=[]
+
+    for i in range(len(dj2)):
+        words = dj2[i].split(",")  # Splitting at the comma delimiter
+        #print(f"Date: {words[0]}")
+        Date.append(words[0])
+            
+        #print(f"Time: {words[1]}")
+        Time.append(words[1])
+            
+        #print(f"Casualities: {words[2]}")
+        Killed.append(words[2])
+        Injured.append(words[3])
+        Location.append(words[4])
+        Road_Characteristic.append(words[5])
+        Pedestrian_Involved.append(words[6])
+        #Vehicles_involved.append(words[7])
+
+    #### Probable type of final dataframe:
+    df2["Accident Date"]=Date
+    df2["Time"]=Time
+    df2["Killed"]=Killed
+    df2["Injured"]=Injured
+    df2["Location"]=Location
+    df2["Road_Characteristic"]=Road_Characteristic
+    df2["Pedestrian_Involved"]=Pedestrian_Involved
+    df2["Vehicles_involved"]=vehicles
+    df3=df2.drop(columns=['Description','Date + Desc','Report Type'])
+    return df3
+    
+
+

Prothom_alo_fully_scraped.py

@@ -0,0 +1,80 @@
+def get_data(number):
+    print("Running Prothom_alo_fully_scraped")
+    ##Necessary imports
+    from selenium import webdriver
+    from selenium.webdriver import chrome
+    from selenium.webdriver import ChromeOptions
+    options = ChromeOptions()
+    options.add_argument("--headless=new")
+    driver = webdriver.Chrome(options=options)
+    ## Finding Elements by XPATH
+    from selenium.webdriver.common.by import By
+
+    driver.get("https://en.prothomalo.com/search?q=road%20accident%20dhaka",)
+
+    import time
+    news_list=[]
+    news_link=[]
+    l=0
+    for i in range(number):
+        if i<15:
+            
+            txt=driver.find_element('xpath',f'/html/body/div/div[6]/div/div/div[1]/div[3]/div[{i+1}]/div/div/div[2]/div/h3/a')
+            news_list.append(txt.text)
+            news_link.append(txt.get_attribute("href"))
+        else:
+            if (i-15)%10==0:
+                time.sleep(5)
+                last_height = driver.execute_script("return document.body.scrollHeight")
+                driver.execute_script(f"window.scrollTo(0, {last_height-1200})")
+                try:
+                    
+                    driver.find_element('xpath',f'/html/body/div/div[6]/div/div/div[1]/div[3]/div[{i+1}]/span').click()
+                except:
+                    l=1
+                if l==1:
+                    time.sleep(5)
+                    try:
+                        driver.find_element('xpath',f'/html/body/div/div[6]/div/div/div[1]/div[3]/div[{i+1}]').click()
+                    except:
+                        time.sleep(5)
+                        driver.find_element('xpath',f'/html/body/div/div[6]/div/div/div[1]/div[3]/div[{i+1}]').click()
+                    l=0
+                time.sleep(5)
+            txt=driver.find_element('xpath',f'/html/body/div/div[6]/div/div/div[1]/div[3]/div[{i+1}]/div/div/div[2]/div/h3/a')
+            news_list.append(txt.text)
+            news_link.append(txt.get_attribute("href"))   
+
+    ###### Scraping Publish Date and Description ######
+
+    publish_date=[]
+    text=[]
+    for i in range (len(news_link)):
+        driver.get(news_link[i])
+        try:
+            publish_date.append(driver.find_element('xpath','/html/body/div/div[6]/div/div/div/div[1]/div[1]/div[1]/div[2]/div[2]/div[1]/time/span').text)
+            tmp=""
+            elements = driver.find_elements(By.TAG_NAME, 'p')
+            for e in elements:
+                tmp=tmp+e.text
+            text.append(tmp)
+        except:
+            publish_date.append("Not available")
+            text.append("Not Available")
+        time.sleep(3)
+
+    #### Converting the list to a pandas dataframe by converting the list to a dictionary  ###
+    dict={'News Title':news_list,'News Link':news_link,'Publish Date':publish_date, 'Description':text}
+    import pandas as pd
+    df=pd.DataFrame(dict)
+    df2=df.copy()
+
+
+    for p in range(len(df2)):
+        if df2['Publish Date'][p]=="Not available":
+            df2.drop([p],inplace=True)
+    #df2.reset_index()
+    df2["Date + Desc"]=df2["Publish Date"] + df2["Description"]
+    df2.reset_index(drop=True,inplace=True)
+    return df2
+    #df3.to_csv('Prothom_Alo_Description.txt',  index=False)

app.py

@@ -0,0 +1,85 @@
+import streamlit as st
+import pandas as pd
+from PIL import Image
+import json
+from streamlit_lottie import st_lottie
+
+##### BUET Logo ###########
+image = Image.open("buet.png")
+new_image = image.resize((100, 100))
+#st.image(new_image)
+st.title("Automated LLM and Web Scrapping based Road Accident Dataset creation from Newspapers")
+
+
+######### Animation ##########
+def load_lottiefile(filepath:str):
+    with open(filepath,"r") as f:
+        return json.load(f)
+lottie_coding=load_lottiefile("animate.json")
+st_lottie(
+        lottie_coding,
+        height=200,
+        
+    )
+
+
+radio_btn1=st.radio("**Choose the newspaper you want to collect news from**",options=("Prothom Alo","Dhaka Tribune","The Daily Star"))
+radio_btn2=st.radio("Choose an LLM model",options=("GPT-3.5 (Medium Cost)","GPT-4 (High Cost)","Llama3 (Free)"))
+
+number = st.number_input("**Enter the number of accident news you want the LLM to go through**",min_value=0,max_value=5000)
+
+if st.button("Generate Dataset"):
+    st.write("**Please wait until the datasest is finished generating. It takes almost 8 sec to process each entry for GPT-4 and 30 sec for GPT-3.5 and Llama3. So, for example, if you entered 15 as input, it will take almost 2 minutes for GPT-4 and 7.5 min for GPT-3.5 and Llama3. The dataset will appear below.**")
+    
+    if radio_btn1=="Prothom Alo":
+        import Prothom_alo_fully_scraped
+        df=Prothom_alo_fully_scraped.get_data(number)
+    elif radio_btn1=="Dhaka Tribune":
+        import Dhaka_Tribune_Fully_Scraped
+        df=Dhaka_Tribune_Fully_Scraped.get_data(number)
+    elif radio_btn1== "The Daily Star":
+        import Daily_Star_fully_scraped
+        df=Daily_Star_fully_scraped.get_data(number)
+    if radio_btn2=="GPT-4 (High Cost)":
+        import LLM_automation_GPT
+        df2=LLM_automation_GPT.create_data(df)
+    elif radio_btn2=="Llama3 (Free)":
+        import LLM_automation_Groq
+        df2=LLM_automation_Groq.create_data(df)
+    elif radio_btn2=="GPT-3.5 (Medium Cost)":
+        import LLM_automation_GPT35
+        df2=LLM_automation_GPT35.create_data(df)
+    st.dataframe(df2)
+    print(len(df))
+    
+    
+#st.write("""
+#                    **Developed by:**\n
+        
+#                    *MD Thamed Bin Zaman Chowdhury, Student ID: 1904184,*\n
+#                    *Department of Civil Engineering, BUET*\n
+#                    *E-mail: zamanthamed@gmail.com*
+#         """)
+
+
+st.write("--------")
+st.write("**Modules and packages used to develop the program:**")
+
+######## Other Logos ################
+p=125
+image2 = Image.open("pandas.png")
+new_image2 = image2.resize((p, p))
+image3 = Image.open("numpy.png")
+new_image3 = image3.resize((p, p))
+image4 = Image.open("selenium_webdriver.jpeg")
+new_image4 = image4.resize((p, p))
+image5 = Image.open("streamlit.png")
+new_image5 = image5.resize((p, p))
+image6 = Image.open("openai.png")
+new_image6 = image6.resize((p, p))
+image7 = Image.open("llama3.jpeg")
+new_image7 = image7.resize((p, p))
+image8 = Image.open("langchain.png")
+new_image8 = image8.resize((p, p))
+
+st.image([new_image2, new_image3,new_image4,new_image5,new_image6,new_image7,new_image8])

packages.txt

@@ -0,0 +1 @@
+chromium

requirements.txt

@@ -0,0 +1,14 @@
+setuptools==70.0.0
+langchain_community==0.2.6
+langchain_core==0.2.10
+langchain_groq==0.1.5
+langchain_openai==0.1.10
+newspaper3k==0.2.8
+pandas==2.2.2
+python-dotenv==1.0.1
+selenium==4.22.0
+streamlit==1.35.0
+lxml==5.2.2
+lxml_html_clean==0.1.1
+webdriver-manager==4.0.1
+streamlit-lottie==0.0.5