Update get_real_home_listing.py

get_real_home_listing.py

@@ -1,111 +1,138 @@
-from selenium import webdriver
-from selenium.webdriver.chrome.service import Service
-from selenium.webdriver.chrome.options import Options
-from selenium.webdriver.common.by import By
-from selenium.webdriver.support.ui import WebDriverWait
-from selenium.webdriver.support import expected_conditions as EC
-from selenium.common.exceptions import NoSuchElementException
-from selenium.webdriver.chrome.service import Service
-from webdriver_manager.chrome import ChromeDriverManager
-import json
-import pandas as pd
-import requests
-from bs4 import BeautifulSoup
-
-def format_price(price):
-    price = str(price)[0:-2]
-    clean_price = price.replace('.', '')
-    return int(clean_price)
-
-
-def get_home_listings(selected_il, price_value):
-    chrome_options = Options()
-    chrome_options.add_argument('--headless')  
-    chrome_options.add_argument('--no-sandbox') 
-    chrome_options.add_argument('--disable-dev-shm-usage') 
-    chrome_options.add_argument('--disable-gpu')  
-    chrome_options.add_argument('--remote-debugging-port=9222') 
-    chrome_options.binary_location = "/usr/bin/chromium"
-    
-    service = Service(ChromeDriverManager().install())
-    driver = webdriver.Chrome(service=service, options=chrome_options)
-
-    driver.get('https://www.emlakjet.com/')
-
-    search_input = WebDriverWait(driver, 10).until(
-        EC.presence_of_element_located((By.XPATH, '//*[@id="headlessui-tabs-panel-:r9:"]/div/div[2]/div/div/div/input'))
-    )
-    search_input.send_keys(f'{selected_il}')
-    dropdown_button = driver.find_element(By.XPATH, '//*[@id="headlessui-listbox-button-:rh:"]')
-    dropdown_button.click()
-
-    # burada tahmin edilen değerin 500.000 TL altı ve üstü aralığında arama yapılacak
-    price_value = format_price(price_value)
-    lower_bound = price_value - 500000
-    upper_bound = price_value + 500000
-
-    first_input = WebDriverWait(driver, 10).until(
-        EC.presence_of_element_located((By.XPATH, '//*[@id="headlessui-listbox-options-:ri:"]/ul[1]/div[1]/div/div[1]/input'))
-    )
-    first_input.clear()
-    first_input.send_keys(str(lower_bound))
-
-    second_input = WebDriverWait(driver, 10).until(
-        EC.presence_of_element_located((By.XPATH, '//*[@id="headlessui-listbox-options-:ri:"]/ul[2]/div[1]/div/div[1]/input'))
-    )
-    second_input.clear()
-    second_input.send_keys(str(upper_bound))
-
-    find_button = WebDriverWait(driver, 10).until(
-        EC.element_to_be_clickable((By.XPATH, '//*[@id="headlessui-tabs-panel-:r9:"]/div/div[5]/div/button'))
-    )
-    find_button.click()
-
-    i = 1
-    data = []
-
-    while i<=10:
-        WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'//*[@id="content-wrapper"]/div[1]/div[4]/div[2]/div[3]/div[{""+str(i)+""}]/div/a')))
-        link = driver.find_element(By.XPATH, f'//*[@id="content-wrapper"]/div[1]/div[4]/div[2]/div[3]/div[{""+str(i)+""}]/div/a')
-        driver.get(link.get_attribute('href'))
-        detail_url = driver.current_url
-        WebDriverWait(driver, 3).until(
-            EC.presence_of_element_located((By.ID, "ilan-hakkinda"))
-        )
-        
-        try:
-            ul = driver.find_element(By.XPATH, '//*[@id="ilan-hakkinda"]/div/div/ul')
-            list_items = ul.find_elements(By.TAG_NAME, 'li')
-            
-            details = {}
-            for item in list_items:
-                try:
-                    key = item.find_element(By.CLASS_NAME, 'styles_key__VqMhC').text
-                    value = item.find_element(By.CLASS_NAME, 'styles_value__3QmL3').text
-                    details[key] = value
-                except NoSuchElementException:
-                    continue
-
-            title = driver.find_element(By.XPATH, '//*[@id="content-wrapper"]/div[2]/div[1]/div/h1').text
-            resim_url = driver.find_element(By.XPATH, '//*[@id="content-wrapper"]/div[2]/div[2]/div[2]/img').get_attribute('src')
-            fiyat = driver.find_element(By.XPATH, '//*[@id="genel-bakis"]/div[1]/div[1]/div[1]/div/span').text
-            fiyat = int(fiyat.replace('.','').replace('TL',''))
-            details['url'] = detail_url
-            details['title'] = title
-            details['resim_url'] = resim_url
-            details['price'] = fiyat
-            
-
-
-            data.append(details)
-
-        except NoSuchElementException as e:
-            print(f"Element not found: {e}")
-        except Exception as e:
-            print(f"An error occurred: {e}")
-
-        driver.execute_script("window.history.go(-1)")
-        i += 1
-        
-    driver.quit()
-    return data
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import NoSuchElementException
+from selenium.webdriver.chrome.service import Service
+from webdriver_manager.chrome import ChromeDriverManager
+import json
+import pandas as pd
+import requests
+from bs4 import BeautifulSoup
+
+def install_chrome():
+    if not os.path.exists("/usr/bin/google-chrome"):
+        print("Installing Google Chrome...")
+        subprocess.run(
+            "wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | sudo apt-key add - && "
+            "echo 'deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main' | sudo tee /etc/apt/sources.list.d/google-chrome.list && "
+            "sudo apt update && sudo apt install -y google-chrome-stable",
+            shell=True,
+            check=True,
+        )
+
+def install_chromedriver():
+    if not os.path.exists("/usr/local/bin/chromedriver"):
+        print("Installing ChromeDriver...")
+        subprocess.run(
+            "wget -O /usr/local/bin/chromedriver https://storage.googleapis.com/chrome-for-testing-public/133.0.6943.98/linux64/chromedriver-linux64.zip && "
+            "unzip /usr/local/bin/chromedriver -d /usr/local/bin/ && "
+            "chmod +x /usr/local/bin/chromedriver",
+            shell=True,
+            check=True,
+        )
+
+
+def format_price(price):
+    price = str(price)[0:-2]
+    clean_price = price.replace('.', '')
+    return int(clean_price)
+
+
+def get_home_listings(selected_il, price_value):
+
+    install_chrome()
+    install_chromedriver()
+    chrome_options = Options()
+    chrome_options.add_argument('--headless')  
+    chrome_options.add_argument('--no-sandbox') 
+    chrome_options.add_argument('--disable-dev-shm-usage') 
+    chrome_options.add_argument('--disable-gpu')  
+    chrome_options.add_argument('--remote-debugging-port=9222') 
+    
+    
+    
+    
+    service = Service("/usr/local/bin/chromedriver")
+    driver = webdriver.Chrome(service=service, options=chrome_options)
+    driver.get('https://www.emlakjet.com/')
+
+    search_input = WebDriverWait(driver, 10).until(
+        EC.presence_of_element_located((By.XPATH, '//*[@id="headlessui-tabs-panel-:r9:"]/div/div[2]/div/div/div/input'))
+    )
+    search_input.send_keys(f'{selected_il}')
+    dropdown_button = driver.find_element(By.XPATH, '//*[@id="headlessui-listbox-button-:rh:"]')
+    dropdown_button.click()
+
+    # burada tahmin edilen değerin 500.000 TL altı ve üstü aralığında arama yapılacak
+    price_value = format_price(price_value)
+    lower_bound = price_value - 500000
+    upper_bound = price_value + 500000
+
+    first_input = WebDriverWait(driver, 10).until(
+        EC.presence_of_element_located((By.XPATH, '//*[@id="headlessui-listbox-options-:ri:"]/ul[1]/div[1]/div/div[1]/input'))
+    )
+    first_input.clear()
+    first_input.send_keys(str(lower_bound))
+
+    second_input = WebDriverWait(driver, 10).until(
+        EC.presence_of_element_located((By.XPATH, '//*[@id="headlessui-listbox-options-:ri:"]/ul[2]/div[1]/div/div[1]/input'))
+    )
+    second_input.clear()
+    second_input.send_keys(str(upper_bound))
+
+    find_button = WebDriverWait(driver, 10).until(
+        EC.element_to_be_clickable((By.XPATH, '//*[@id="headlessui-tabs-panel-:r9:"]/div/div[5]/div/button'))
+    )
+    find_button.click()
+
+    i = 1
+    data = []
+
+    while i<=10:
+        WebDriverWait(driver,10).until(EC.presence_of_element_located((By.XPATH, f'//*[@id="content-wrapper"]/div[1]/div[4]/div[2]/div[3]/div[{""+str(i)+""}]/div/a')))
+        link = driver.find_element(By.XPATH, f'//*[@id="content-wrapper"]/div[1]/div[4]/div[2]/div[3]/div[{""+str(i)+""}]/div/a')
+        driver.get(link.get_attribute('href'))
+        detail_url = driver.current_url
+        WebDriverWait(driver, 3).until(
+            EC.presence_of_element_located((By.ID, "ilan-hakkinda"))
+        )
+        
+        try:
+            ul = driver.find_element(By.XPATH, '//*[@id="ilan-hakkinda"]/div/div/ul')
+            list_items = ul.find_elements(By.TAG_NAME, 'li')
+            
+            details = {}
+            for item in list_items:
+                try:
+                    key = item.find_element(By.CLASS_NAME, 'styles_key__VqMhC').text
+                    value = item.find_element(By.CLASS_NAME, 'styles_value__3QmL3').text
+                    details[key] = value
+                except NoSuchElementException:
+                    continue
+
+            title = driver.find_element(By.XPATH, '//*[@id="content-wrapper"]/div[2]/div[1]/div/h1').text
+            resim_url = driver.find_element(By.XPATH, '//*[@id="content-wrapper"]/div[2]/div[2]/div[2]/img').get_attribute('src')
+            fiyat = driver.find_element(By.XPATH, '//*[@id="genel-bakis"]/div[1]/div[1]/div[1]/div/span').text
+            fiyat = int(fiyat.replace('.','').replace('TL',''))
+            details['url'] = detail_url
+            details['title'] = title
+            details['resim_url'] = resim_url
+            details['price'] = fiyat
+            
+
+
+            data.append(details)
+
+        except NoSuchElementException as e:
+            print(f"Element not found: {e}")
+        except Exception as e:
+            print(f"An error occurred: {e}")
+
+        driver.execute_script("window.history.go(-1)")
+        i += 1
+        
+    driver.quit()
+    return data