Merge remote-tracking branch 'refs/remotes/origin/main'

crawl/crawl

@@ -42,10 +42,10 @@ def sanitize_filename(filename):
         str: The sanitized filename.
     """
     # Remove invalid characters for Windows file names
-    return re.sub(r'[<>:"/\\|?*]', '', filename)
+    return re.sub(r'[<>:"/\\|?*]', "", filename)
 
 
-def download_image(session, image_url, save_dir):
+def download_image(session, image_url, save_dir, base_url):
     """
     Download an image from a given URL and save it to the specified directory.
 
@@ -56,19 +56,25 @@ def download_image(session, image_url, save_dir):
             The URL of the image to download.
         save_dir (str):
             The directory to save the downloaded image.
+        base_url (str):
+            The base URL of the page being crawled.
     """
     try:
         # Ensure the URL has a scheme
-        if not re.match(r'^https?://', image_url):
-            image_url = 'https://' + image_url.lstrip('/')
+        if image_url.startswith(".."):
+            from urllib.parse import urljoin
 
-        image_filename = os.path.basename(image_url).split('?')[0]
+            image_url = urljoin(base_url, image_url)
+        elif not re.match(r"^https?://", image_url):
+            image_url = "https://" + image_url.lstrip("/")
+
+        image_filename = os.path.basename(image_url).split("?")[0]
         sanitized_image_filename = sanitize_filename(image_filename)
         image_path = os.path.join(save_dir, sanitized_image_filename)
 
         response = session.get(image_url, stream=True)
         response.raise_for_status()
-        with open(image_path, 'wb') as image_file:
+        with open(image_path, "wb") as image_file:
             for chunk in response.iter_content(chunk_size=8192):
                 image_file.write(chunk)
         print(f"Saved image: {image_path}")
@@ -90,15 +96,15 @@ def save_result(target_url):
     """
     crawler = create_crawler()
     result = crawler.run(url=target_url)
-    title = result.metadata.get('title', 'untitled')
+    title = result.metadata.get("title", "untitled")
     sanitized_title = sanitize_filename(title).replace(" ", "_")
-    
+
     # Choose the appropriate base path based on the operating system
     if platform.system() == "Windows":
         base_path = "E:\\knowledgebase\\Saved Websites\\"
     else:
         base_path = "/home/kade/saved_websites/"
-    
+
     save_dir = os.path.join(base_path, sanitized_title)
     os.makedirs(save_dir, exist_ok=True)
 
@@ -109,38 +115,45 @@ def save_result(target_url):
     print(f"Saved markdown to {save_path}")
 
     # Save images in parallel
-    if 'images' in result.media and isinstance(result.media['images'], list):
+    if "images" in result.media and isinstance(result.media["images"], list):
         session = requests.Session()
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
-                          'AppleWebKit/537.36 (KHTML, like Gecko) '
-                          'Chrome/91.0.4472.124 Safari/537.36',
-            'Referer': target_url,
-            'Accept': ('image/avif,image/webp,image/apng,image/svg+xml,'
-                       'image/*,*/*;q=0.8'),
-            'Accept-Language': 'en-US,en;q=0.9',
-            'Sec-Fetch-Dest': 'image',
-            'Sec-Fetch-Mode': 'no-cors',
-            'Sec-Fetch-Site': 'cross-site',
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/91.0.4472.124 Safari/537.36",
+            "Referer": target_url,
+            "Accept": (
+                "image/avif,image/webp,image/apng,image/svg+xml," "image/*,*/*;q=0.8"
+            ),
+            "Accept-Language": "en-US,en;q=0.9",
+            "Sec-Fetch-Dest": "image",
+            "Sec-Fetch-Mode": "no-cors",
+            "Sec-Fetch-Site": "cross-site",
         }
         session.headers.update(headers)
 
         with ThreadPoolExecutor(max_workers=5) as executor:
             futures = []
-            for image_data in result.media['images']:
-                if 'src' in image_data:
-                    futures.append(executor.submit(download_image,
-                                                   session,
-                                                   image_data['src'],
-                                                   save_dir))
+            for image_data in result.media["images"]:
+                if "src" in image_data:
+                    futures.append(
+                        executor.submit(
+                            download_image,
+                            session,
+                            image_data["src"],
+                            save_dir,
+                            target_url,
+                        )
+                    )
 
             for future in as_completed(futures):
                 future.result()
 
 
 if __name__ == "__main__":
-    if len(sys.argv) != 2:
-        print("Usage: python crawl.py <URL>")
+    if len(sys.argv) < 2:
+        print("Usage: python crawl.py <URL1> <URL2> ... <URLn>")
     else:
-        url = sys.argv[1]
-        save_result(url)
+        urls = sys.argv[1:]
+        for url in urls:
+            save_result(url)