Update app.py

app.py

@@ -14,8 +14,11 @@ def process_url(url, *args):
         return "", "Please enter a URL"
     
     try:
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url.strip('/')
+        
         output_file = token_hex(6)
-        adv.crawl(url, f"{output_file}.jsonl")
+        adv.crawl(url, f"{output_file}.jsonl", follow_links=True)
         crawl_df = pd.read_json(f"{output_file}.jsonl", lines=True)
         
         all_links = []