update crawl

Signed-off-by: Balazs Horvath <acsipont@gmail.com>

crawl/crawl

@@ -14,9 +14,24 @@ import os
 import re
 import platform
 from concurrent.futures import ThreadPoolExecutor, as_completed
-
+import time
+import argparse
+from urllib.parse import urljoin
 import requests
-from crawl4ai import WebCrawler
+try:
+    from crawl4ai import WebCrawler  # type: ignore
+except ImportError as exc:
+    raise ImportError(
+        "The module 'crawl4ai' could not be imported. Please ensure it is "
+        "installed and accessible."
+    ) from exc
+
+# Check if the current Conda environment is "crawl"
+conda_env = os.environ.get('CONDA_DEFAULT_ENV')
+if conda_env != 'crawl':
+    print(f"Error: The current Conda environment is '{conda_env}'. "
+          "Please activate the 'crawl' environment.")
+    sys.exit(1)
 
 
 def create_crawler():
@@ -62,8 +77,6 @@ def download_image(session, image_url, save_dir, base_url):
     try:
         # Ensure the URL has a scheme
         if image_url.startswith(".."):
-            from urllib.parse import urljoin
-
             image_url = urljoin(base_url, image_url)
         elif not re.match(r"^https?://", image_url):
             image_url = "https://" + image_url.lstrip("/")
@@ -78,10 +91,10 @@ def download_image(session, image_url, save_dir, base_url):
             for chunk in response.iter_content(chunk_size=8192):
                 image_file.write(chunk)
         print(f"Saved image: {image_path}")
-    except requests.RequestException as e:
-        print(f"Error downloading image {image_url}: {str(e)}")
-    except IOError as e:
-        print(f"Error saving image {image_url}: {str(e)}")
+    except requests.RequestException as req_err:
+        print(f"Error downloading image {image_url}: {str(req_err)}")
+    except IOError as io_err:
+        print(f"Error saving image {image_url}: {str(io_err)}")
 
 
 def save_result(target_url):
@@ -96,6 +109,8 @@ def save_result(target_url):
     """
     crawler = create_crawler()
     result = crawler.run(url=target_url)
+    if result is None:
+        raise ValueError(f"Failed to crawl {target_url}")
     title = result.metadata.get("title", "untitled")
     sanitized_title = sanitize_filename(title).replace(" ", "_")
 
@@ -123,7 +138,8 @@ def save_result(target_url):
             "Chrome/91.0.4472.124 Safari/537.36",
             "Referer": target_url,
             "Accept": (
-                "image/avif,image/webp,image/apng,image/svg+xml," "image/*,*/*;q=0.8"
+                "image/avif,image/webp,image/apng,image/svg+xml,"
+                "image/*,*/*;q=0.8"
             ),
             "Accept-Language": "en-US,en;q=0.9",
             "Sec-Fetch-Dest": "image",
@@ -151,9 +167,22 @@ def save_result(target_url):
 
 
 if __name__ == "__main__":
-    if len(sys.argv) < 2:
-        print("Usage: python crawl.py <URL1> <URL2> ... <URLn>")
-    else:
-        urls = sys.argv[1:]
-        for url in urls:
-            save_result(url)
+    parser = argparse.ArgumentParser(
+        description="Web Crawler and Content Saver"
+    )
+    parser.add_argument(
+        "urls",
+        nargs="+",
+        help="List of URLs to crawl"
+    )
+    args = parser.parse_args()
+
+    for url in args.urls:
+        while True:
+            try:
+                save_result(url)
+                break
+            except (AttributeError, ValueError) as e:
+                print(f"[ERROR] 🚫 Failed to crawl {url}, error: {str(e)}")
+                print("Retrying in 3 seconds...")
+                time.sleep(3)