Spaces:
Running
Running
| import gradio as gr | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import re | |
| from urllib.parse import urljoin, urlparse | |
| import asyncio | |
| from collections import defaultdict | |
| import unicodedata | |
| import logging | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| class WebsiteCrawler: | |
| def __init__(self, max_depth=3, max_pages=50): | |
| self.max_depth = max_depth | |
| self.max_pages = max_pages | |
| self.visited_urls = set() | |
| self.url_metadata = defaultdict(dict) | |
| self.homepage_metadata = None | |
| self.headers = { | |
| "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36" | |
| } | |
| def determine_category_importance(self, url, title, desc): | |
| """Improved category detection""" | |
| url_lower = url.lower() | |
| path = urlparse(url).path.lower() | |
| # Homepage | |
| if path == "/" or path == "": | |
| return "Main", 10 | |
| # Documentation and Features | |
| if any(x in url_lower for x in ['/docs', '/documentation', '/features', '/pricing']): | |
| return "Documentation", 8 | |
| # API | |
| elif any(x in url_lower for x in ['/api', '/developer', 'developers']): | |
| return "API", 8 | |
| # About/Company | |
| elif any(x in url_lower for x in ['/about', '/company', '/partners', '/stories']): | |
| return "About", 7 | |
| # News and Updates | |
| elif any(x in url_lower for x in ['/news', '/blog', '/releases', '/academy']): | |
| return "News", 5 | |
| # Tools and Features | |
| elif any(x in url_lower for x in ['/tools', '/features', '/website', '/keyword']): | |
| return "Tools", 6 | |
| return "Optional", 1 | |
| def clean_text(self, text, is_title=False): | |
| """Improved text cleaning""" | |
| if not text or len(text.strip()) < 2: | |
| return "" | |
| # Normalize unicode characters | |
| text = unicodedata.normalize("NFKD", text) | |
| text = re.sub(r"[^\x00-\x7F]+", "", text) | |
| # Remove any template variables/placeholders | |
| text = re.sub(r'\{\{.*?\}\}', '', text) | |
| text = re.sub(r'\{\%.*?\%\}', '', text) | |
| text = re.sub(r'\${.*?\}', '', text) | |
| if is_title: | |
| # Remove common suffixes and fragments for titles | |
| text = re.sub(r'^\s*Welcome to\s+', '', text) | |
| text = re.sub(r'\s*[\|\-#:•].*', '', text) | |
| text = re.sub(r'\s+Homepage$', '', text, flags=re.IGNORECASE) | |
| # Handle overly generic titles | |
| if text.lower() in ['features', 'home', 'homepage', 'welcome']: | |
| return "" | |
| # Only return if we have meaningful text | |
| cleaned = " ".join(text.split()).strip() | |
| if len(cleaned.split()) < 2 and not is_title: # Allow single-word titles | |
| return "" | |
| return cleaned | |
| async def process_homepage(self, url): | |
| """Specifically process the homepage to extract key metadata""" | |
| try: | |
| response = requests.get(url, headers=self.headers, timeout=10) | |
| response.encoding = "utf-8" | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # Extract site name with more fallbacks | |
| site_name = None | |
| # Try meta tags first | |
| site_meta = soup.find("meta", property="og:site_name") | |
| if site_meta and site_meta.get("content"): | |
| site_name = site_meta["content"] | |
| # Try structured data | |
| if not site_name: | |
| schema = soup.find("script", type="application/ld+json") | |
| if schema: | |
| try: | |
| import json | |
| data = json.loads(schema.string) | |
| if isinstance(data, dict): | |
| site_name = data.get("name") or data.get("organizationName") | |
| except: | |
| pass | |
| # Try title tag | |
| if not site_name: | |
| title_tag = soup.find("title") | |
| if title_tag: | |
| site_name = title_tag.text.split('|')[0].strip() | |
| # Last resort - use domain name | |
| if not site_name: | |
| site_name = urlparse(url).netloc.split('.')[0].capitalize() | |
| # Get homepage description | |
| description = self.extract_homepage_description(soup) | |
| self.homepage_metadata = { | |
| "site_name": self.clean_text(site_name, is_title=True), | |
| "description": description | |
| } | |
| except Exception as e: | |
| logger.error(f"Error processing homepage {url}: {str(e)}") | |
| self.homepage_metadata = { | |
| "site_name": urlparse(url).netloc.split('.')[0].capitalize(), | |
| "description": None | |
| } | |
| def clean_description(self, desc): | |
| """Clean description text""" | |
| if not desc: | |
| return "" | |
| # Remove leading dashes, hyphens, or colons | |
| desc = re.sub(r"^[-:\s]+", "", desc) | |
| # Remove any strings that are just "Editors", "APIs", etc. | |
| if len(desc.split()) <= 1: | |
| return "" | |
| return desc.strip() | |
| def is_duplicate_content(self, desc, title, url): | |
| """Improved duplicate/translation detection""" | |
| if not desc or not title: | |
| return False | |
| # Skip common translation paths | |
| translation_indicators = [ | |
| '/welcome', '/bienvenue', '/willkommen', '/benvenuto', | |
| '/tervetuloa', '/bienvenido', '/velkommen', '/welkom' | |
| ] | |
| if any(indicator in url.lower() for indicator in translation_indicators): | |
| return True | |
| # Check for similar content length and patterns | |
| for existing_metadata in self.url_metadata.values(): | |
| existing_desc = existing_metadata.get("description", "") | |
| if not existing_desc: | |
| continue | |
| # If descriptions are very similar in length, likely a translation | |
| if (abs(len(desc) - len(existing_desc)) < 20 and | |
| len(desc) > 50): # Only check substantial descriptions | |
| return True | |
| return False | |
| def extract_homepage_description(self, soup): | |
| """Extract description from homepage with multiple fallbacks""" | |
| # Try meta description first | |
| meta_desc = soup.find("meta", {"name": "description"}) | |
| if meta_desc and meta_desc.get("content"): | |
| desc = meta_desc["content"] | |
| if desc and len(desc.strip()) > 20: | |
| return self.clean_text(desc) | |
| # Try OpenGraph description | |
| og_desc = soup.find("meta", property="og:description") | |
| if og_desc and og_desc.get("content"): | |
| desc = og_desc["content"] | |
| if desc and len(desc.strip()) > 20: | |
| return self.clean_text(desc) | |
| # Try first significant paragraph | |
| for p in soup.find_all("p"): | |
| text = p.get_text().strip() | |
| if len(text) > 50 and not any(x in text.lower() for x in ["cookie", "accept", "privacy"]): | |
| return self.clean_text(text) | |
| # Try main content area if exists | |
| main = soup.find("main") | |
| if main: | |
| first_p = main.find("p") | |
| if first_p: | |
| text = first_p.get_text().strip() | |
| if len(text) > 50: | |
| return self.clean_text(text) | |
| return None | |
| async def crawl_page(self, url, depth, base_domain): | |
| """Crawl a single page and extract information""" | |
| if ( | |
| depth > self.max_depth | |
| or url in self.visited_urls | |
| or len(self.visited_urls) >= self.max_pages | |
| ): | |
| return [] | |
| try: | |
| await asyncio.sleep(1) # Be polite to servers | |
| response = requests.get(url, headers=self.headers, timeout=10) | |
| response.encoding = "utf-8" | |
| self.visited_urls.add(url) | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # Extract title with fallbacks | |
| title = None | |
| meta_title = soup.find("meta", property="og:title") | |
| if meta_title and meta_title.get("content"): | |
| title = meta_title["content"] | |
| if not title: | |
| title_tag = soup.find("title") | |
| if title_tag: | |
| title = title_tag.text | |
| if not title: | |
| h1_tag = soup.find("h1") | |
| if h1_tag: | |
| title = h1_tag.text | |
| if not title: | |
| title = url.split("/")[-1] | |
| title = self.clean_text(title, is_title=True) | |
| # Extract description with fallbacks | |
| desc = None | |
| meta_desc = soup.find("meta", {"name": "description"}) | |
| if meta_desc and meta_desc.get("content"): | |
| desc = meta_desc["content"] | |
| if not desc: | |
| og_desc = soup.find("meta", property="og:description") | |
| if og_desc and og_desc.get("content"): | |
| desc = og_desc["content"] | |
| if not desc: | |
| first_p = soup.find("p") | |
| if first_p: | |
| desc = first_p.text | |
| desc = self.clean_text(desc) if desc else "" | |
| # Skip if it's duplicate content | |
| if self.is_duplicate_content(desc, title, url): | |
| return [] | |
| # Determine category and importance | |
| category, importance = self.determine_category_importance(url, title, desc) | |
| # Store metadata | |
| clean_url = re.sub(r"#.*", "", url).rstrip("/") | |
| if title and len(title.strip()) > 0: # Only store if we have a valid title | |
| self.url_metadata[clean_url] = { | |
| "title": title, | |
| "description": desc, | |
| "category": category, | |
| "importance": importance, | |
| } | |
| # Find links | |
| links = [] | |
| for a in soup.find_all("a", href=True): | |
| href = a["href"] | |
| if not any( | |
| x in href.lower() | |
| for x in ["javascript:", "mailto:", ".pdf", ".jpg", ".png", ".gif"] | |
| ): | |
| next_url = urljoin(url, href) | |
| if urlparse(next_url).netloc == base_domain: | |
| links.append(next_url) | |
| return links | |
| except Exception as e: | |
| logger.error(f"Error crawling {url}: {str(e)}") | |
| return [] | |
| async def process_homepage(self, url): | |
| """Specifically process the homepage to extract key metadata""" | |
| try: | |
| response = requests.get(url, headers=self.headers, timeout=10) | |
| response.encoding = "utf-8" | |
| soup = BeautifulSoup(response.text, "html.parser") | |
| # Extract site name with fallbacks | |
| site_name = None | |
| site_meta = soup.find("meta", property="og:site_name") | |
| if site_meta and site_meta.get("content"): | |
| site_name = site_meta["content"] | |
| if not site_name: | |
| site_name = soup.find("title").text if soup.find("title") else None | |
| if not site_name: | |
| site_name = urlparse(url).netloc.split('.')[0].capitalize() | |
| # Get homepage description | |
| description = self.extract_homepage_description(soup) | |
| self.homepage_metadata = { | |
| "site_name": self.clean_text(site_name, is_title=True), | |
| "description": description | |
| } | |
| except Exception as e: | |
| logger.error(f"Error processing homepage {url}: {str(e)}") | |
| self.homepage_metadata = { | |
| "site_name": urlparse(url).netloc.split('.')[0].capitalize(), | |
| "description": None | |
| } | |
| async def crawl_website(self, start_url): | |
| """Crawl website starting from the given URL""" | |
| # First process the homepage | |
| await self.process_homepage(start_url) | |
| base_domain = urlparse(start_url).netloc | |
| queue = [(start_url, 0)] | |
| seen = {start_url} | |
| while queue and len(self.visited_urls) < self.max_pages: | |
| current_url, depth = queue.pop(0) | |
| if depth > self.max_depth: | |
| continue | |
| links = await self.crawl_page(current_url, depth, base_domain) | |
| for link in links: | |
| if link not in seen and urlparse(link).netloc == base_domain: | |
| seen.add(link) | |
| queue.append((link, depth + 1)) | |
| def generate_llms_txt(self): | |
| """Generate llms.txt content""" | |
| if not self.url_metadata: | |
| return "No content was found to generate llms.txt" | |
| # Sort URLs by importance and remove duplicates | |
| sorted_urls = [] | |
| seen_titles = set() | |
| for url, metadata in sorted( | |
| self.url_metadata.items(), | |
| key=lambda x: (x[1]["importance"], x[0]), | |
| reverse=True, | |
| ): | |
| if metadata["title"] not in seen_titles: | |
| sorted_urls.append((url, metadata)) | |
| seen_titles.add(metadata["title"]) | |
| if not sorted_urls: | |
| return "No valid content was found" | |
| # Generate content | |
| content = [] | |
| # Use homepage metadata for main title and description | |
| main_title = self.homepage_metadata.get("site_name", "Welcome") | |
| homepage_description = self.homepage_metadata.get("description") | |
| content.append(f"# {main_title}") | |
| if homepage_description: | |
| content.append(f"\n> {homepage_description}") | |
| elif len(sorted_urls) > 0: | |
| # Fallback to first good description from content if no homepage description | |
| for _, metadata in sorted_urls: | |
| desc = self.clean_description(metadata["description"]) | |
| if desc and len(desc) > 20 and "null" not in desc.lower(): | |
| content.append(f"\n> {desc}") | |
| break | |
| # Group by category | |
| categories = defaultdict(list) | |
| for url, metadata in sorted_urls: | |
| if metadata["title"] and url: | |
| categories[metadata["category"]].append((url, metadata)) | |
| # Add sections in a logical order | |
| category_order = [ | |
| "Main", | |
| "Documentation", | |
| "API", | |
| "Tools", | |
| "About", | |
| "News", | |
| "Optional" | |
| ] | |
| for category in category_order: | |
| if category in categories: | |
| content.append(f"\n## {category}") | |
| # Sort links within category by importance and description length | |
| category_links = sorted( | |
| categories[category], | |
| key=lambda x: (-len(x[1]["description"] or ""), x[1]["title"]) | |
| ) | |
| links = [] | |
| for url, metadata in category_links: | |
| title = metadata["title"].strip() | |
| desc = self.clean_description(metadata["description"]) | |
| if desc: | |
| links.append(f"- [{title}]({url}): {desc}") | |
| else: | |
| links.append(f"- [{title}]({url})") | |
| content.append("\n".join(links)) | |
| return "\n".join(content) | |
| async def process_url(url, max_depth, max_pages): | |
| """Process URL and generate llms.txt""" | |
| try: | |
| # Add https:// if not present | |
| if not url.startswith(("http://", "https://")): | |
| url = "https://" + url | |
| # Validate URL | |
| result = urlparse(url) | |
| if not all([result.scheme, result.netloc]): | |
| return "", "Invalid URL format. Please enter a valid URL." | |
| # Process website | |
| crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages)) | |
| await crawler.crawl_website(url) | |
| content = crawler.generate_llms_txt() | |
| return content, f"Successfully crawled {len(crawler.visited_urls)} pages." | |
| except Exception as e: | |
| return "", f"Error: {str(e)}" | |
| # Create Gradio interface | |
| theme = gr.themes.Soft(primary_hue="blue", font="Open Sans") | |
| with gr.Blocks( | |
| theme=theme, | |
| css=""" | |
| @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap'); | |
| .gradio-container { | |
| font-family: 'Open Sans', sans-serif !important; | |
| } | |
| .gr-button { | |
| font-family: 'Open Sans', sans-serif !important; | |
| font-weight: 600 !important; | |
| } | |
| .primary-btn { | |
| background-color: #2436d4 !important; | |
| color: white !important; | |
| } | |
| .primary-btn:hover { | |
| background-color: #1c2aa8 !important; | |
| } | |
| [data-testid="textbox"] { | |
| font-family: 'Open Sans', sans-serif !important; | |
| } | |
| .gr-padded { | |
| font-family: 'Open Sans', sans-serif !important; | |
| } | |
| .gr-input { | |
| font-family: 'Open Sans', sans-serif !important; | |
| } | |
| .gr-label { | |
| font-family: 'Open Sans', sans-serif !important; | |
| } | |
| """, | |
| ) as iface: | |
| gr.Markdown("# llms.txt Generator") | |
| gr.Markdown("Generate an llms.txt file from a website following the specification.") | |
| with gr.Row(): | |
| url_input = gr.Textbox( | |
| label="Website URL", | |
| placeholder="Enter the website URL (e.g., example.com)", | |
| info="The URL will be automatically prefixed with https:// if not provided", | |
| ) | |
| with gr.Row(): | |
| with gr.Column(): | |
| depth_input = gr.Slider( | |
| minimum=1, maximum=5, value=3, step=1, label="Maximum Crawl Depth" | |
| ) | |
| with gr.Column(): | |
| pages_input = gr.Slider( | |
| minimum=10, maximum=100, value=50, step=10, label="Maximum Pages" | |
| ) | |
| generate_btn = gr.Button("Generate llms.txt", variant="primary") | |
| output = gr.Textbox( | |
| label="Generated llms.txt Content", | |
| lines=20, | |
| show_copy_button=True, | |
| container=True, | |
| ) | |
| status = gr.Textbox(label="Status") | |
| generate_btn.click( | |
| fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)), | |
| inputs=[url_input, depth_input, pages_input], | |
| outputs=[output, status], | |
| ) | |
| if __name__ == "__main__": | |
| iface.launch() | |