import gradio as gr import requests from bs4 import BeautifulSoup import re from urllib.parse import urljoin, urlparse import asyncio import aiohttp from collections import defaultdict import unicodedata import logging import ssl logging.basicConfig(level=logging.INFO) logger = logging.getLogger(__name__) class WebsiteCrawler: def __init__(self, max_depth=3, max_pages=50): self.max_depth = max_depth self.max_pages = max_pages self.visited_urls = set() self.url_metadata = defaultdict(dict) self.homepage_metadata = None self.headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8", "Accept-Language": "en-US,en;q=0.5", "Accept-Encoding": "gzip, deflate, br", "DNT": "1", "Connection": "keep-alive", "Upgrade-Insecure-Requests": "1", } def determine_category_importance(self, url, title, desc): """Improved category detection""" url_lower = url.lower() path = urlparse(url).path.lower() # Homepage if path == "/" or path == "": return "Main", 10 # Documentation and Help if any( x in url_lower for x in [ "/docs", "/documentation", "/faq", "/help", "frequently-asked-questions", ] ): return "Documentation", 8 # API and Developer elif any(x in url_lower for x in ["/api", "/developer", "developers"]): return "API", 8 # About/Company pages elif any( x in url_lower for x in [ "/about", "/company", "/references", "/work-with-us", "careers", "/team", "/contact", "/about-us", ] ): return "About", 7 # News and Events elif any( x in url_lower for x in [ "/news", "/blog", "/events", "/press", "research", "power-of", "latest", ] ): return "News", 5 # Tools and Services elif any( x in url_lower for x in [ "/tools", "/quote", "/pricing", "/services", "/translate", "/order", "/buy", ] ): return "Tools", 6 # Check if URL path contains non-ASCII or percent-encoded characters if bool(re.search(r"[^\x00-\x7F]", path)) or bool( re.search(r"%[0-9A-F]{2}", path) ): return "Optional", 0 return "Optional", 1 def is_duplicate_content(self, desc, title, url): """Improved duplicate/translation detection""" if not desc or not title: return False # Skip non-latin character URLs or URLs with percent-encoded non-ASCII if bool(re.search(r"[^\x00-\x7F]", url)) or bool( re.search(r"%[0-9A-F]{2}", url) ): return True # Skip common translation paths translation_indicators = [ "/welcome", "/bienvenue", "/willkommen", "/benvenuto", "/tervetuloa", "/bienvenido", "/velkommen", "/welkom", "translate.com/", "/translate/", "/translation/", ] if any(indicator in url.lower() for indicator in translation_indicators): url_path = urlparse(url).path.lower() if url_path != "/": # Don't skip homepage return True # Check for similar content length and patterns for existing_metadata in self.url_metadata.values(): existing_desc = existing_metadata.get("description", "") existing_title = existing_metadata.get("title", "") if not existing_desc or not existing_title: continue # If descriptions are very similar in length, likely a translation if ( abs(len(desc) - len(existing_desc)) < 20 and len(desc) > 50 and desc != existing_desc ): # Allow exact duplicates for main page return True return False def clean_text(self, text, is_title=False): """Improved text cleaning""" if not text or len(text.strip()) < 2: return "" # Normalize unicode characters text = unicodedata.normalize("NFKD", text) text = re.sub(r"[^\x00-\x7F]+", "", text) # Remove any template variables/placeholders text = re.sub(r"\{\{.*?\}\}", "", text) text = re.sub(r"\{\%.*?\%\}", "", text) text = re.sub(r"\${.*?\}", "", text) if is_title: # Remove common suffixes and fragments for titles text = re.sub(r"^\s*Welcome to\s+", "", text) text = re.sub(r"\s*[\|\-#:•].*", "", text) text = re.sub(r"\s+Homepage$", "", text, flags=re.IGNORECASE) # Handle overly generic titles if text.lower() in ["features", "home", "homepage", "welcome"]: return "" # Only return if we have meaningful text cleaned = " ".join(text.split()).strip() if len(cleaned.split()) < 2 and not is_title: # Allow single-word titles return "" return cleaned def clean_description(self, desc): """Clean description text""" if not desc: return "" # Remove leading dashes, hyphens, or colons desc = re.sub(r"^[-:\s]+", "", desc) # Remove any strings that are just "Editors", "APIs", etc. if len(desc.split()) <= 1: return "" return desc.strip() def extract_homepage_description(self, soup): """Extract description from homepage with multiple fallbacks""" # Try meta description first meta_desc = soup.find("meta", {"name": "description"}) if meta_desc and meta_desc.get("content"): desc = meta_desc["content"] if desc and len(desc.strip()) > 20: return self.clean_text(desc) # Try OpenGraph description og_desc = soup.find("meta", property="og:description") if og_desc and og_desc.get("content"): desc = og_desc["content"] if desc and len(desc.strip()) > 20: return self.clean_text(desc) # Try first significant paragraph for p in soup.find_all("p"): text = p.get_text().strip() if len(text) > 50 and not any( x in text.lower() for x in ["cookie", "accept", "privacy"] ): return self.clean_text(text) # Try main content area if exists main = soup.find("main") if main: first_p = main.find("p") if first_p: text = first_p.get_text().strip() if len(text) > 50: return self.clean_text(text) return None async def crawl_page(self, url, depth, base_domain): """Crawl a single page and extract information""" if ( depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages ): return [] try: await asyncio.sleep(1) # Be polite to servers async with aiohttp.ClientSession() as session: async with session.get( url, headers=self.headers, allow_redirects=True ) as response: if response.status == 403: # Try with alternative headers alt_headers = { "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", } async with session.get( url, headers=alt_headers, allow_redirects=True ) as retry_response: if retry_response.status != 200: return [] text = await retry_response.text() elif response.status != 200: return [] else: text = await response.text() self.visited_urls.add(url) soup = BeautifulSoup(text, "html.parser") # Extract title with fallbacks title = None meta_title = soup.find("meta", property="og:title") if meta_title and meta_title.get("content"): title = meta_title["content"] if not title: title_tag = soup.find("title") if title_tag: title = title_tag.text if not title: h1_tag = soup.find("h1") if h1_tag: title = h1_tag.text if not title: title = url.split("/")[-1] title = self.clean_text(title, is_title=True) # Extract description with fallbacks desc = None meta_desc = soup.find("meta", {"name": "description"}) if meta_desc and meta_desc.get("content"): desc = meta_desc["content"] if not desc: og_desc = soup.find("meta", property="og:description") if og_desc and og_desc.get("content"): desc = og_desc["content"] if not desc: first_p = soup.find("p") if first_p: desc = first_p.text desc = self.clean_text(desc) if desc else "" # Skip if it's duplicate content if self.is_duplicate_content(desc, title, url): return [] # Determine category and importance category, importance = self.determine_category_importance( url, title, desc ) # Store metadata clean_url = re.sub(r"#.*", "", url).rstrip("/") if ( title and len(title.strip()) > 0 ): # Only store if we have a valid title logger.info( f"Storing metadata for {clean_url}: {title[:30]}..." ) self.url_metadata[clean_url] = { "title": title, "description": desc, "category": category, "importance": importance, } # Find links links = [] for a in soup.find_all("a", href=True): href = a["href"] if not any( x in href.lower() for x in [ "javascript:", "mailto:", ".pdf", ".jpg", ".png", ".gif", ] ): next_url = urljoin(url, href) if urlparse(next_url).netloc == base_domain: links.append(next_url) return links except Exception as e: logger.error(f"Error crawling {url}: {str(e)}") return [] async def process_homepage(self, url): """Specifically process the homepage to extract key metadata""" try: # Configure SSL context ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE connector = aiohttp.TCPConnector(ssl=ssl_context) timeout = aiohttp.ClientTimeout(total=30) async with aiohttp.ClientSession( connector=connector, timeout=timeout ) as session: async with session.get( url, headers=self.headers, allow_redirects=True ) as response: if response.status != 200: raise Exception( f"Failed to fetch homepage: status {response.status}" ) try: text = await response.text() except UnicodeDecodeError: text = await response.read() text = text.decode("utf-8", errors="ignore") soup = BeautifulSoup(text, "html.parser") # Extract site name with more fallbacks site_name = None # Try meta tags first site_meta = soup.find("meta", property="og:site_name") if site_meta and site_meta.get("content"): site_name = site_meta["content"] # Try structured data if not site_name: schema = soup.find("script", type="application/ld+json") if schema: try: import json data = json.loads(schema.string) if isinstance(data, dict): site_name = data.get("name") or data.get( "organizationName" ) except: pass # Try title tag if not site_name: title_tag = soup.find("title") if title_tag: site_name = title_tag.text.split("|")[0].strip() # Last resort - use domain name if not site_name: site_name = urlparse(url).netloc.split(".")[0].capitalize() # Get homepage description description = self.extract_homepage_description(soup) self.homepage_metadata = { "site_name": self.clean_text(site_name, is_title=True), "description": description, } except Exception as e: logger.error(f"Error processing homepage {url}: {str(e)}") self.homepage_metadata = { "site_name": urlparse(url).netloc.split(".")[0].capitalize(), "description": None, } async def crawl_website(self, start_url): """Crawl website starting from the given URL""" try: # First process the homepage logger.info(f"Processing homepage: {start_url}") await self.process_homepage(start_url) base_domain = urlparse(start_url).netloc queue = [(start_url, 0)] seen = {start_url} while queue and len(self.visited_urls) < self.max_pages: current_url, depth = queue.pop(0) if depth > self.max_depth: continue logger.info(f"Crawling page: {current_url} (depth: {depth})") links = await self.crawl_page(current_url, depth, base_domain) logger.info(f"Found {len(links)} links on {current_url}") for link in links: if link not in seen and urlparse(link).netloc == base_domain: seen.add(link) queue.append((link, depth + 1)) logger.info(f"Crawl completed. Visited {len(self.visited_urls)} pages") except Exception as e: logger.error(f"Error during crawl: {str(e)}") raise def generate_llms_txt(self): """Generate llms.txt content""" logger.info(f"Starting generate_llms_txt with {len(self.url_metadata)} URLs") if not self.url_metadata: logger.error("No URL metadata found") return "No content was found to generate llms.txt" # Sort URLs by importance and remove duplicates sorted_urls = [] seen_titles = set() for url, metadata in sorted( self.url_metadata.items(), key=lambda x: (x[1]["importance"], x[0]), reverse=True, ): if metadata["title"] not in seen_titles: sorted_urls.append((url, metadata)) seen_titles.add(metadata["title"]) logger.info(f"Found {len(sorted_urls)} unique URLs after deduplication") if not sorted_urls: logger.error("No valid URLs found after sorting") return "No valid content was found" # Generate content content = [] # Use homepage metadata for main title and description main_title = self.homepage_metadata.get("site_name", "Welcome") homepage_description = self.homepage_metadata.get("description") logger.info(f"Homepage title: {main_title}") logger.info(f"Homepage description: {homepage_description}") content.append(f"# {main_title}") if homepage_description: content.append(f"\n> {homepage_description}") elif len(sorted_urls) > 0: # Fallback to first good description from content if no homepage description for _, metadata in sorted_urls: desc = self.clean_description(metadata["description"]) if desc and len(desc) > 20 and "null" not in desc.lower(): content.append(f"\n> {desc}") break # Group by category categories = defaultdict(list) for url, metadata in sorted_urls: if metadata["title"] and url: categories[metadata["category"]].append((url, metadata)) logger.info(f"Categories found: {list(categories.keys())}") # Add sections in a logical order category_order = [ "Main", "Documentation", "API", "Tools", "About", "News", "Optional", ] # Only show Main section if it has content different from the homepage description if "Main" in categories: main_content = categories["Main"] if ( len(main_content) == 1 and main_content[0][1]["description"] == homepage_description ): logger.info("Removing duplicate Main content") del categories["Main"] for category in category_order: if category in categories and categories[category]: logger.info( f"Processing category {category} with {len(categories[category])} items" ) content.append(f"\n## {category}") # Sort links within category by importance and description length category_links = sorted( categories[category], key=lambda x: (-len(x[1]["description"] or ""), x[1]["title"]), ) links = [] seen_desc = set() # Avoid duplicate descriptions within category for url, metadata in category_links: title = metadata["title"].strip() desc = self.clean_description(metadata["description"]) # Skip if description is duplicate within category if desc in seen_desc: continue seen_desc.add(desc) if desc: links.append(f"- [{title}]({url}): {desc}") else: links.append(f"- [{title}]({url})") content.append("\n".join(links)) final_content = "\n".join(content) logger.info(f"Generated content length: {len(final_content)}") return final_content async def process_url(url, max_depth, max_pages): """Process URL and generate llms.txt""" try: # Add https:// if not present if not url.startswith(("http://", "https://")): url = "https://" + url # Validate URL result = urlparse(url) if not all([result.scheme, result.netloc]): return "", "Invalid URL format. Please enter a valid URL." logger.info(f"Starting crawl of {url}") # Process website crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages)) await crawler.crawl_website(url) logger.info("Generating llms.txt content") content = crawler.generate_llms_txt() if not content or content.strip() == "": return "", "No content was generated. Check the logs for details." return content, f"Successfully crawled {len(crawler.visited_urls)} pages." except Exception as e: logger.error(f"Error processing URL {url}: {str(e)}") return "", f"Error: {str(e)}" # Create Gradio interface theme = gr.themes.Soft(primary_hue="blue", font="Open Sans") with gr.Blocks( theme=theme, css=""" @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap'); .gradio-container { font-family: 'Open Sans', sans-serif !important; } .gr-button { font-family: 'Open Sans', sans-serif !important; font-weight: 600 !important; } .primary-btn { background-color: #2436d4 !important; color: white !important; } .primary-btn:hover { background-color: #1c2aa8 !important; } [data-testid="textbox"] { font-family: 'Open Sans', sans-serif !important; } .gr-padded { font-family: 'Open Sans', sans-serif !important; } .gr-input { font-family: 'Open Sans', sans-serif !important; } .gr-label { font-family: 'Open Sans', sans-serif !important; } """, ) as iface: with gr.Row(): url_input = gr.Textbox( label="Website URL", placeholder="Enter the website URL (e.g., example.com)", info="The URL will be automatically prefixed with https:// if not provided", ) with gr.Row(): with gr.Column(): depth_input = gr.Slider( minimum=1, maximum=5, value=3, step=1, label="Maximum Crawl Depth" ) with gr.Column(): pages_input = gr.Slider( minimum=10, maximum=100, value=50, step=10, label="Maximum Pages" ) generate_btn = gr.Button("Generate llms.txt", variant="primary") output = gr.Textbox( label="Generated llms.txt Content", lines=20, show_copy_button=True, container=True, ) status = gr.Textbox(label="Status") generate_btn.click( fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)), inputs=[url_input, depth_input, pages_input], outputs=[output, status], ) if __name__ == "__main__": iface.launch()