{ "cells": [ { "metadata": { "ExecuteTime": { "end_time": "2024-10-26T16:28:29.519384Z", "start_time": "2024-10-26T16:28:29.506673Z" } }, "cell_type": "code", "source": [ "from fake_headers import Headers\n", "\n", "headers = Headers(headers=True).generate()\n", "headers" ], "id": "c60b4d771c2e0a21", "outputs": [ { "data": { "text/plain": [ "{'Accept': '*/*',\n", " 'Connection': 'keep-alive',\n", " 'User-Agent': 'Mozilla/5.0 (X11; Linux i686 on x86_64; rv:60.3.0) Gecko/20100101 Firefox/60.3.0',\n", " 'Accept-Language': 'en-US;q=0.5,en;q=0.3',\n", " 'DNT': '1',\n", " 'Referer': 'https://google.com'}" ] }, "execution_count": 1, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 1 }, { "metadata": { "ExecuteTime": { "end_time": "2024-10-26T16:28:29.719882Z", "start_time": "2024-10-26T16:28:29.531148Z" } }, "cell_type": "code", "source": [ "from selenium.webdriver.chrome.options import Options\n", "from selenium import webdriver\n", "from selenium.webdriver.support.ui import WebDriverWait\n", "from selenium.webdriver.support import expected_conditions as EC\n", "from selenium.webdriver.common.by import By\n", "from bs4 import BeautifulSoup\n", "import time\n", "\n", "\n", "def scroll_and_wait(driver, scroll_pause_time=2):\n", " \"\"\"\n", " Scroll the page gradually and wait for images to load\n", " \"\"\"\n", " # Get scroll height\n", " last_height = driver.execute_script(\"return document.body.scrollHeight\")\n", "\n", " while True:\n", " # Scroll down gradually\n", " for i in range(10):\n", " driver.execute_script(f\"window.scrollTo(0, {(i + 1) * (last_height / 10)});\")\n", " time.sleep(0.5) # Short pause between each scroll step\n", "\n", " # Wait for new images to load\n", " time.sleep(scroll_pause_time)\n", "\n", " # Calculate new scroll height and compare with last scroll height\n", " new_height = driver.execute_script(\"return document.body.scrollHeight\")\n", " if new_height == last_height:\n", " break\n", " last_height = new_height\n", "\n", "\n", "def wait_for_images(driver, timeout=10):\n", " \"\"\"\n", " Wait for images to load and become visible\n", " \"\"\"\n", " try:\n", " # Wait for all image elements to be present\n", " WebDriverWait(driver, timeout).until(\n", " EC.presence_of_all_elements_located((By.TAG_NAME, \"img\"))\n", " )\n", "\n", " # Get all image elements\n", " images = driver.find_elements(By.TAG_NAME, \"img\")\n", "\n", " # Wait for images to load\n", " for img in images:\n", " try:\n", " WebDriverWait(driver, 2).until(\n", " lambda d: img.get_attribute('complete') == 'true' and\n", " img.get_attribute('naturalHeight') != '0'\n", " )\n", " except:\n", " continue # Skip images that don't load within timeout\n", "\n", " except Exception as e:\n", " print(f\"Warning: Not all images could be loaded: {e}\")" ], "id": "11933d956e20b6b8", "outputs": [], "execution_count": 2 }, { "metadata": { "ExecuteTime": { "end_time": "2024-10-26T16:29:00.452959Z", "start_time": "2024-10-26T16:28:29.721884Z" } }, "cell_type": "code", "source": [ "chrome_options = Options()\n", "# chrome_options.add_argument(\"--headless\")\n", "# chrome_options.add_argument(\"--disable-gpu\")\n", "# chrome_options.add_argument(\"--no-sandbox\")\n", "# chrome_options.add_argument(\"--disable-dev-shm-usage\")\n", "\n", "# Add fake headers\n", "for key, value in headers.items():\n", " chrome_options.add_argument(f'--{key.lower()}={value}')\n", "\n", "# Additional configurations to appear more human-like\n", "chrome_options.add_argument(\"--disable-blink-features=AutomationControlled\")\n", "chrome_options.add_argument(\"--window-size=1920,1080\")\n", "\n", "# Enable images in headless mode\n", "chrome_options.add_argument(\"--force-device-scale-factor=1\")\n", "chrome_options.add_argument(\"--high-dpi-support=1\")\n", "\n", "# Privacy and fingerprinting prevention\n", "chrome_options.add_argument(\"--disable-blink-features\")\n", "chrome_options.add_argument(\"--disable-infobars\")\n", "chrome_options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n", "chrome_options.add_experimental_option(\"useAutomationExtension\", False)\n", "\n", "# Enable JavaScript\n", "chrome_options.add_argument(\"--enable-javascript\")\n", "\n", "driver = webdriver.Chrome(options=chrome_options)\n", "\n", "driver.execute_cdp_cmd(\"Page.addScriptToEvaluateOnNewDocument\", {\n", " \"source\": \"\"\"\n", " Object.defineProperty(navigator, 'webdriver', {\n", " get: () => undefined\n", " })\n", " \"\"\"\n", "})\n", "\n", "products_url = \"https://www.target.com/s?searchTerm=Peach&tref=typeahead%7Cterm%7CPeach%7C%7C%7Chistory\"\n", "driver.get(products_url)\n", "\n", "time.sleep(3)\n", "\n", "# Scroll and wait for content\n", "scroll_and_wait(driver)\n", "\n", "# Wait for images to load\n", "wait_for_images(driver)\n", "\n", "time.sleep(2)\n", "\n", "soup = BeautifulSoup(driver.page_source, \"html.parser\")\n", "driver.quit()" ], "id": "ac14cff825f0887f", "outputs": [], "execution_count": 3 }, { "metadata": { "ExecuteTime": { "end_time": "2024-10-26T16:54:58.190620Z", "start_time": "2024-10-26T16:54:58.165031Z" } }, "cell_type": "code", "source": [ "from urllib.parse import urljoin\n", "import json\n", "from collections import Counter\n", "\n", "\n", "def get_element_signature(element):\n", " \"\"\"\n", " Create a signature for an element based on its structure.\n", " \"\"\"\n", " signature = {\n", " 'tag': element.name,\n", " 'classes': tuple(sorted(element.get('class', []))),\n", " 'child_tags': tuple(sorted(child.name for child in element.find_all(recursive=False) if child.name)),\n", " 'has_image': bool(element.find('img')),\n", " 'has_price': bool(any(c in element.get_text() for c in '$€£¥')),\n", " 'has_link': bool(element.find('a')),\n", " }\n", " return str(signature)\n", "\n", "\n", "def analyze_children_similarity(element):\n", " \"\"\"\n", " Analyze how similar the direct children of an element are.\n", " \"\"\"\n", " if not element.contents:\n", " return 0, 0\n", "\n", " # Get signatures for all direct children that are elements (have a tag name)\n", " child_signatures = [\n", " get_element_signature(child)\n", " for child in element.find_all(recursive=False)\n", " if child.name\n", " ]\n", "\n", " if not child_signatures:\n", " return 0, 0\n", "\n", " # Count how many times each signature appears and get the most common one\n", " signature_counts = Counter(child_signatures)\n", " most_common_sig, most_common_count = signature_counts.most_common(1)[0]\n", " similarity_score = most_common_count / len(child_signatures)\n", "\n", " return similarity_score, most_common_count\n", "\n", "\n", "def count_images_in_element(element):\n", " \"\"\"\n", " Count all images within an element, including nested ones.\n", " \"\"\"\n", " return len(element.find_all('img', recursive=True))\n", "\n", "\n", "def get_element_identifier(element):\n", " \"\"\"\n", " Create a unique identifier for an element including tag and classes.\n", " \"\"\"\n", " identifier = element.name\n", " if element.get('class'):\n", " identifier += f\" .{' .'.join(element['class'])}\"\n", " if element.get('id'):\n", " identifier += f\" #{element['id']}\"\n", " return identifier\n", "\n", "\n", "def convert_relative_urls(soup, base_url):\n", " \"\"\"\n", " Convert all relative URLs in the soup object to absolute URLs.\n", " \"\"\"\n", " for tag in soup.find_all(href=True):\n", " tag['href'] = urljoin(base_url, tag['href'])\n", " for tag in soup.find_all(src=True):\n", " tag['src'] = urljoin(base_url, tag['src'])\n", " for tag in soup.find_all(attrs={'data-src': True}):\n", " tag['data-src'] = urljoin(base_url, tag['data-src'])\n", " return soup\n", "\n", "\n", "def find_image_rich_parents(soup, base_url, min_children=4, min_similarity=0.7):\n", " \"\"\"\n", " Find elements containing images and return both sorted list and detailed top element info.\n", " \"\"\"\n", " # Convert relative URLs to absolute\n", " soup = convert_relative_urls(soup, base_url)\n", "\n", " # Collect potential container elements with their scores\n", " elements_with_scores = []\n", " for element in soup.find_all():\n", " if element.name in ['div', 'ul', 'section', 'main']:\n", " similarity_score, similar_children_count = analyze_children_similarity(element)\n", " image_count = count_images_in_element(element)\n", "\n", " if similar_children_count >= min_children and similarity_score >= min_similarity and image_count > 0:\n", " # Calculate combined score based on similarity and image count\n", " combined_score = (similarity_score * similar_children_count * image_count)\n", " elements_with_scores.append((element, image_count, combined_score))\n", "\n", " if not elements_with_scores:\n", " return [], {\"error\": \"No elements with images found\"}, \"\"\n", "\n", " # Sort by combined score\n", " elements_with_scores.sort(key=lambda x: x[2], reverse=True)\n", "\n", " # Process elements for sorted list output\n", " sorted_elements = []\n", " for element, image_count, _ in elements_with_scores:\n", " sorted_elements.append((get_element_identifier(element), image_count))\n", "\n", " # Get top element (one with highest combined score)\n", " top_element = elements_with_scores[0][0]\n", "\n", " # Separate child elements with images\n", " products = []\n", " for child in top_element.find_all(recursive=False):\n", " if child.name: # Skip text nodes\n", " product_info = {\n", " \"html_content\": str(child),\n", " \"images\": []\n", " }\n", "\n", " # Get all images within this product\n", " for img in child.find_all('img', recursive=True):\n", " image_info = {\n", " \"src\": img.get('src', 'No source'),\n", " \"alt\": img.get('alt', 'No alt text')\n", " }\n", " product_info[\"images\"].append(image_info)\n", "\n", " products.append(product_info)\n", "\n", " print(len(products))\n", "\n", " # Create result dictionary for top element \n", " top_element_info = {\n", " \"parent\": {\n", " \"tag\": top_element.name,\n", " \"identifier\": get_element_identifier(top_element),\n", " \"classes\": top_element.get('class', []),\n", " \"id\": top_element.get('id', None)\n", " },\n", " \"products_count\": len(products),\n", " \"products\": products\n", " }\n", "\n", " # Create styled HTML output\n", " style_tag = \"\"\"\n", " \n", " \"\"\"\n", " html_output = style_tag + str(top_element)\n", "\n", " return sorted_elements, json.dumps(top_element_info, indent=2), html_output\n", "\n", "\n", "def print_results(element_list):\n", " \"\"\"\n", " Print formatted results.\n", " \"\"\"\n", " print(\"\\nElements Containing Most Images (Lowest Level for Each Count):\")\n", " print(\"-\" * 70)\n", " print(\"Rank Element Tag & Classes Image Count\")\n", " print(\"-\" * 70)\n", "\n", " for rank, element in enumerate(element_list, 1):\n", " tag_info, count = element\n", " rank_str = f\"{rank}.\"\n", " rank_str = rank_str.ljust(5)\n", " tag_info_padded = tag_info.ljust(45)\n", " print(f\"{rank_str} {tag_info_padded} {count}\")" ], "id": "3830f2e224e84798", "outputs": [], "execution_count": 11 }, { "metadata": {}, "cell_type": "markdown", "source": "", "id": "80fa7f140d4da0a2" }, { "metadata": { "ExecuteTime": { "end_time": "2024-10-26T16:55:03.174631Z", "start_time": "2024-10-26T16:55:02.976453Z" } }, "cell_type": "code", "source": [ "base_url = products_url.rsplit('/', 1)[0]\n", "sorted_elements, top_element_info, html_output = find_image_rich_parents(soup, base_url)\n", "\n", "# Print sorted list\n", "print_results(sorted_elements)\n", "\n", "with open(\"output1.json\", \"w\") as file:\n", " file.write(top_element_info)\n", "\n", "with open(\"output1.html\", \"w\") as file:\n", " file.write(html_output)" ], "id": "20b0b8cd238de02d", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "28\n", "\n", "Elements Containing Most Images (Lowest Level for Each Count):\n", "----------------------------------------------------------------------\n", "Rank Element Tag & Classes Image Count\n", "----------------------------------------------------------------------\n", "1. div .sc-5da3fdcc-0 .cqdDWw 51\n", "2. div 1\n" ] } ], "execution_count": 12 }, { "metadata": {}, "cell_type": "markdown", "source": "", "id": "1465ddb6bce2981c" } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 5 }