Update app.py

app.py

@@ -9,53 +9,110 @@ import os
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
 
-def process_url(url, *args):
+def filter_links_by_type(link_df, link_types):
+    """Filter links based on selected types"""
+    if not link_types or "All links" in link_types:
+        return link_df
+    
+    filtered_links = pd.DataFrame(columns=link_df.columns)
+    for link_type in link_types:
+        if "<header>" in link_type:
+            header_links = link_df[link_df['source'].str.contains('<header', case=False, na=False)]
+            filtered_links = pd.concat([filtered_links, header_links])
+        elif "<nav>" in link_type:
+            nav_links = link_df[link_df['source'].str.contains('<nav', case=False, na=False)]
+            filtered_links = pd.concat([filtered_links, nav_links])
+        elif "<footer>" in link_type:
+            footer_links = link_df[link_df['source'].str.contains('<footer', case=False, na=False)]
+            filtered_links = pd.concat([filtered_links, footer_links])
+    
+    return filtered_links.drop_duplicates()
+
+def process_url(url, link_types):
+    """Process URL and generate llms.txt content"""
     if not url:
         return "", "Please enter a URL"
     
     try:
-        if not url.startswith(('http://', 'https://')):
-            url = 'https://' + url.strip('/')
-        
+        if not url.startswith(("http://", "https://")):
+            url = "https://" + url
+
+        # Generate unique filename for this crawl
         output_file = token_hex(6)
-        adv.crawl(url, f"{output_file}.jsonl", follow_links=True)
-        crawl_df = pd.read_json(f"{output_file}.jsonl", lines=True)
-        
-        all_links = []
-        title = crawl_df['title'].values[0]
-        meta_desc = crawl_df['meta_desc'].values[0]
-        link_df = adv.crawlytics.links(crawl_df)
-        
-        for link, text in link_df[['link', 'text']].values:
-            if text.strip():
-                text = re.sub(r"\n+", " ", text.strip(), re.DOTALL)
-                text = re.sub(r"\s{3,}", " ", text)
-                all_links.append("\n".join(['## ' + text, f"[{text}]({link})"]))
-
-        separator = "\n\n"
-        final_content = f"# {title}\n> {meta_desc}\n{separator.join(all_links)}"
-
-        if os.path.exists(f"{output_file}.jsonl"):
-            os.remove(f"{output_file}.jsonl")
-
-        return final_content, "Successfully crawled website."
+        jsonl_path = f"{output_file}.jsonl"
+
+        try:
+            # Perform the crawl using advertools
+            adv.crawl(url, jsonl_path)
+            
+            # Read the crawl results
+            crawl_df = pd.read_json(jsonl_path, lines=True)
+            
+            # Extract title and meta description
+            title = crawl_df['title'].values[0]
+            meta_desc = crawl_df['meta_desc'].values[0]
+            
+            # Process links using advertools
+            link_df = adv.crawlytics.links(crawl_df)
+            
+            # Filter links based on selected types
+            if link_types:
+                link_df = filter_links_by_type(link_df, link_types)
+            
+            # Generate all links content
+            all_links = []
+            for link, text in link_df[['link', 'text']].values:
+                if text and text.strip():
+                    text = re.sub(r"\n+", " ", text.strip(), re.DOTALL)
+                    text = re.sub(r"\s{3,}", " ", text)
+                    all_links.append("\n".join(['## ' + text, f"[{text}]({link})"]))
+
+            # Generate final content
+            final_content = f"# {title}\n> {meta_desc}\n{chr(10).join(all_links)}"
+
+        finally:
+            # Cleanup temporary file
+            if os.path.exists(jsonl_path):
+                os.remove(jsonl_path)
+
+        return final_content, f"Successfully crawled website. Found {len(all_links)} links."
     
     except Exception as e:
         logger.error(f"Error processing URL {url}: {str(e)}")
         return "", f"Error: {str(e)}"
 
+# Custom CSS for Open Sans font and color theme
 css = """
 @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@300;400;600;700&display=swap');
-body { font-family: 'Open Sans', sans-serif !important; }
+
+body {
+    font-family: 'Open Sans', sans-serif !important;
+}
+
+.primary-btn {
+    background-color: #3452db !important;
+}
+
+.primary-btn:hover {
+    background-color: #2a41af !important;
+}
 """
 
+# Create custom theme with specific color
 theme = gr.themes.Soft(
     primary_hue=gr.themes.colors.Color(
         name="blue",
-        c50="#eef1ff", c100="#e0e5ff", c200="#c3cbff",
-        c300="#a5b2ff", c400="#8798ff", c500="#6a7eff",
-        c600="#3452db", c700="#2a41af", c800="#1f3183",
-        c900="#152156", c950="#0a102b",
+        c50="#eef1ff",
+        c100="#e0e5ff",
+        c200="#c3cbff",
+        c300="#a5b2ff",
+        c400="#8798ff",
+        c500="#6a7eff",
+        c600="#3452db",  # Our main color
+        c700="#2a41af",
+        c800="#1f3183",
+        c900="#152156",
+        c950="#0a102b",
     )
 )
 
@@ -67,7 +124,7 @@ with gr.Blocks(theme=theme, css=css) as iface:
         url_input = gr.Textbox(
             label="Enter the home page of a website:",
             placeholder="example: https://example.com",
-            lines=1
+            lines=1,
         )
 
     with gr.Row():
@@ -79,20 +136,22 @@ with gr.Blocks(theme=theme, css=css) as iface:
         )
 
     with gr.Row():
-        generate_btn = gr.Button("Submit", variant="primary")
+        generate_btn = gr.Button("Submit", variant="primary", elem_classes=["primary-btn"])
 
     with gr.Row():
         output = gr.Textbox(
             label="Generated llms.txt Content",
             lines=20,
-            show_copy_button=True
+            show_copy_button=True,
+            container=True,
         )
         status = gr.Textbox(label="Status", interactive=False)
     
+    # Set up the click event
     generate_btn.click(
         fn=process_url,
         inputs=[url_input, link_types],
-        outputs=[output, status]
+        outputs=[output, status],
     )
 
 if __name__ == "__main__":