import gradio as gr import requests from bs4 import BeautifulSoup import re from urllib.parse import urljoin, urlparse import markdown from concurrent.futures import ThreadPoolExecutor import asyncio from collections import defaultdict import time import logging # Set up logging logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class WebsiteCrawler: def __init__(self, max_depth=3, max_pages=50, timeout=30): self.max_depth = max_depth self.max_pages = max_pages self.timeout = timeout self.visited_urls = set() self.url_content = {} self.url_metadata = defaultdict(dict) self.headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' } def is_valid_url(self, url, base_domain): """Check if URL is valid and belongs to the same domain""" try: parsed = urlparse(url) base_parsed = urlparse(base_domain) return (parsed.netloc == base_parsed.netloc and parsed.scheme in ['http', 'https'] and not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip'))) except: return False def extract_content(self, soup): """Extract meaningful content from HTML""" # Remove script and style elements for element in soup(['script', 'style', 'nav', 'footer', 'header']): element.decompose() # Get main content main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)}) if main_content: return main_content.get_text(strip=True) return soup.get_text(strip=True) def get_page_metadata(self, soup, url): """Extract metadata from the page""" metadata = { 'title': None, 'description': None, 'importance': 0, 'category': 'Optional' } # Title extraction metadata['title'] = ( soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else soup.find('title').text if soup.find('title') else soup.find('h1').text if soup.find('h1') else url.split('/')[-1] ) # Description extraction metadata['description'] = ( soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else "" ) # Calculate importance based on various factors importance = 0 if 'docs' in url.lower() or 'documentation' in url.lower(): importance += 5 metadata['category'] = 'Docs' if 'api' in url.lower(): importance += 4 metadata['category'] = 'API' if 'guide' in url.lower() or 'tutorial' in url.lower(): importance += 3 metadata['category'] = 'Guides' if 'example' in url.lower(): importance += 2 metadata['category'] = 'Examples' if 'blog' in url.lower(): importance += 1 metadata['category'] = 'Blog' metadata['importance'] = importance return metadata async def crawl_page(self, url, depth, base_domain): """Crawl a single page and extract information""" if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages: return [] try: response = requests.get(url, headers=self.headers, timeout=self.timeout) response.raise_for_status() self.visited_urls.add(url) soup = BeautifulSoup(response.text, 'html.parser') content = self.extract_content(soup) metadata = self.get_page_metadata(soup, url) self.url_content[url] = content self.url_metadata[url] = metadata # Find all links links = [] for a in soup.find_all('a', href=True): next_url = urljoin(url, a['href']) if self.is_valid_url(next_url, base_domain): links.append(next_url) return links except Exception as e: logger.error(f"Error crawling {url}: {str(e)}") return [] async def crawl_website(self, start_url): """Crawl website starting from the given URL""" base_domain = start_url queue = [(start_url, 0)] seen = {start_url} while queue and len(self.visited_urls) < self.max_pages: current_url, depth = queue.pop(0) if depth > self.max_depth: continue links = await self.crawl_page(current_url, depth, base_domain) for link in links: if link not in seen: seen.add(link) queue.append((link, depth + 1)) def generate_llms_txt(self): """Generate llms.txt content from crawled data""" # Sort URLs by importance sorted_urls = sorted( self.url_metadata.items(), key=lambda x: (x[1]['importance'], x[0]), reverse=True ) # Group URLs by category categorized_urls = defaultdict(list) for url, metadata in sorted_urls: categorized_urls[metadata['category']].append((url, metadata)) # Generate content content = [] # Add main title and description if sorted_urls: main_metadata = sorted_urls[0][1] content.append(f"# {main_metadata['title']}\n") content.append(f"> {main_metadata['description']}\n") # Add categorized sections priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional'] for category in priority_order: if category in categorized_urls: content.append(f"\n## {category}\n") for url, metadata in categorized_urls[category]: title = metadata['title'] desc = metadata['description'] if desc: content.append(f"- [{title}]({url}): {desc[:100]}...\n") else: content.append(f"- [{title}]({url})\n") return "\n".join(content) def save_llms_txt(content, save_path="llms.txt"): """Save the generated content to a file""" try: with open(save_path, 'w', encoding='utf-8') as f: f.write(content) return f"Successfully saved to {save_path}" except Exception as e: return f"Error saving file: {str(e)}" async def process_url(url, max_depth, max_pages, save_to_file=False): """Process URL and generate llms.txt""" try: crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages)) await crawler.crawl_website(url) content = crawler.generate_llms_txt() if save_to_file: save_message = save_llms_txt(content) return content, f"Crawled {len(crawler.visited_urls)} pages. {save_message}" return content, f"Crawled {len(crawler.visited_urls)} pages. File not saved (checkbox not selected)" except Exception as e: return "", f"Error: {str(e)}" # Create the Gradio interface with custom CSS for Open Sans font css = """ @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap'); body, .gradio-container { font-family: 'Open Sans', sans-serif !important; } .gr-box { border-radius: 8px !important; border: 1px solid #e5e7eb !important; } .gr-button { font-family: 'Open Sans', sans-serif !important; font-weight: 600 !important; } """ # Create the Gradio interface iface = gr.Interface( fn=lambda url, max_depth, max_pages, save: asyncio.run(process_url(url, max_depth, max_pages, save)), inputs=[ gr.Textbox(label="Website URL", placeholder="Enter the website URL..."), gr.Slider(minimum=1, maximum=5, value=3, step=1, label="Maximum Crawl Depth"), gr.Slider(minimum=10, maximum=100, value=50, step=10, label="Maximum Pages to Crawl"), gr.Checkbox(label="Save to file", value=False) ], outputs=[ gr.Textbox(label="Generated llms.txt Content", lines=20), gr.Textbox(label="Status") ], title="llms.txt Generator", description="Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.", examples=[ ["https://example.com", 3, 50, False], ["https://docs.python.org", 3, 50, True] ], theme=gr.themes.Soft(), css=css ) # Launch the app if __name__ == "__main__": iface.launch()