retry

Signed-off-by: Balazs Horvath <acsipont@gmail.com>

crawl/crawl

@@ -175,14 +175,25 @@ if __name__ == "__main__":
         nargs="+",
         help="List of URLs to crawl"
     )
+    parser.add_argument(
+        "--retry",
+        action="store_true",
+        help="Retry crawling indefinitely every 3 seconds until successful"
+    )
     args = parser.parse_args()
 
     for url in args.urls:
-        while True:
+        if args.retry:
+            while True:
+                try:
+                    save_result(url)
+                    break
+                except (AttributeError, ValueError) as e:
+                    print(f"[ERROR] 🚫 Failed to crawl {url}, error: {str(e)}")
+                    print("Retrying in 3 seconds...")
+                    time.sleep(3)
+        else:
             try:
                 save_result(url)
-                break
             except (AttributeError, ValueError) as e:
                 print(f"[ERROR] 🚫 Failed to crawl {url}, error: {str(e)}")
-                print("Retrying in 3 seconds...")
-                time.sleep(3)