trailing whitespace yay

crawl/crawl

@@ -42,7 +42,7 @@ def sanitize_filename(filename):
         str: The sanitized filename.
     """
     # Remove invalid characters for Windows file names
-    return re.sub(r'[<>:"/\\|?*]', '', filename)
+    return re.sub(r'[<>:"/\\|?*]', "", filename)
 
 
 def download_image(session, image_url, save_dir, base_url):
@@ -61,19 +61,20 @@ def download_image(session, image_url, save_dir, base_url):
     """
     try:
         # Ensure the URL has a scheme
-        if image_url.startswith('..'):
+        if image_url.startswith(".."):
             from urllib.parse import urljoin
+
             image_url = urljoin(base_url, image_url)
-        elif not re.match(r'^https?://', image_url):
-            image_url = 'https://' + image_url.lstrip('/')
+        elif not re.match(r"^https?://", image_url):
+            image_url = "https://" + image_url.lstrip("/")
 
-        image_filename = os.path.basename(image_url).split('?')[0]
+        image_filename = os.path.basename(image_url).split("?")[0]
         sanitized_image_filename = sanitize_filename(image_filename)
         image_path = os.path.join(save_dir, sanitized_image_filename)
 
         response = session.get(image_url, stream=True)
         response.raise_for_status()
-        with open(image_path, 'wb') as image_file:
+        with open(image_path, "wb") as image_file:
             for chunk in response.iter_content(chunk_size=8192):
                 image_file.write(chunk)
         print(f"Saved image: {image_path}")
@@ -95,15 +96,15 @@ def save_result(target_url):
     """
     crawler = create_crawler()
     result = crawler.run(url=target_url)
-    title = result.metadata.get('title', 'untitled')
+    title = result.metadata.get("title", "untitled")
     sanitized_title = sanitize_filename(title).replace(" ", "_")
-    
+
     # Choose the appropriate base path based on the operating system
     if platform.system() == "Windows":
         base_path = "E:\\knowledgebase\\Saved Websites\\"
     else:
         base_path = "/home/kade/saved_websites/"
-    
+
     save_dir = os.path.join(base_path, sanitized_title)
     os.makedirs(save_dir, exist_ok=True)
 
@@ -114,31 +115,36 @@ def save_result(target_url):
     print(f"Saved markdown to {save_path}")
 
     # Save images in parallel
-    if 'images' in result.media and isinstance(result.media['images'], list):
+    if "images" in result.media and isinstance(result.media["images"], list):
         session = requests.Session()
         headers = {
-            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'
-                          'AppleWebKit/537.36 (KHTML, like Gecko) '
-                          'Chrome/91.0.4472.124 Safari/537.36',
-            'Referer': target_url,
-            'Accept': ('image/avif,image/webp,image/apng,image/svg+xml,'
-                       'image/*,*/*;q=0.8'),
-            'Accept-Language': 'en-US,en;q=0.9',
-            'Sec-Fetch-Dest': 'image',
-            'Sec-Fetch-Mode': 'no-cors',
-            'Sec-Fetch-Site': 'cross-site',
+            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64)"
+            "AppleWebKit/537.36 (KHTML, like Gecko) "
+            "Chrome/91.0.4472.124 Safari/537.36",
+            "Referer": target_url,
+            "Accept": (
+                "image/avif,image/webp,image/apng,image/svg+xml," "image/*,*/*;q=0.8"
+            ),
+            "Accept-Language": "en-US,en;q=0.9",
+            "Sec-Fetch-Dest": "image",
+            "Sec-Fetch-Mode": "no-cors",
+            "Sec-Fetch-Site": "cross-site",
         }
         session.headers.update(headers)
 
         with ThreadPoolExecutor(max_workers=5) as executor:
             futures = []
-            for image_data in result.media['images']:
-                if 'src' in image_data:
-                    futures.append(executor.submit(download_image,
-                                                   session,
-                                                   image_data['src'],
-                                                   save_dir,
-                                                   target_url))
+            for image_data in result.media["images"]:
+                if "src" in image_data:
+                    futures.append(
+                        executor.submit(
+                            download_image,
+                            session,
+                            image_data["src"],
+                            save_dir,
+                            target_url,
+                        )
+                    )
 
             for future in as_completed(futures):
                 future.result()