Spaces:
Running
Running
| import gradio as gr | |
| import requests | |
| from bs4 import BeautifulSoup | |
| import re | |
| from urllib.parse import urljoin, urlparse | |
| import markdown | |
| from concurrent.futures import ThreadPoolExecutor | |
| import asyncio | |
| from collections import defaultdict | |
| import time | |
| import logging | |
| import unicodedata | |
| # Set up logging | |
| logging.basicConfig(level=logging.INFO) | |
| logger = logging.getLogger(__name__) | |
| class WebsiteCrawler: | |
| def __init__(self, max_depth=3, max_pages=50, timeout=30): | |
| self.max_depth = max_depth | |
| self.max_pages = max_pages | |
| self.timeout = timeout | |
| self.visited_urls = set() | |
| self.url_content = {} | |
| self.url_metadata = defaultdict(dict) | |
| self.headers = { | |
| 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36' | |
| } | |
| def normalize_text(self, text): | |
| """Normalize text to handle encoding issues""" | |
| if not text: | |
| return "" | |
| # Normalize unicode characters | |
| text = unicodedata.normalize('NFKD', text) | |
| # Replace special quotes and dashes with standard characters | |
| text = text.replace('"', '"').replace('"', '"').replace(''', "'").replace('—', '-') | |
| # Remove any remaining non-ASCII characters | |
| text = text.encode('ascii', 'ignore').decode('ascii') | |
| # Clean up extra whitespace | |
| text = ' '.join(text.split()) | |
| return text | |
| def is_valid_url(self, url, base_domain): | |
| """Check if URL is valid and belongs to the same domain""" | |
| try: | |
| parsed = urlparse(url) | |
| base_parsed = urlparse(base_domain) | |
| return (parsed.netloc == base_parsed.netloc and | |
| parsed.scheme in ['http', 'https'] and | |
| not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip'))) | |
| except: | |
| return False | |
| def extract_content(self, soup): | |
| """Extract meaningful content from HTML""" | |
| # Remove script and style elements | |
| for element in soup(['script', 'style', 'nav', 'footer', 'header']): | |
| element.decompose() | |
| # Get main content | |
| main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)}) | |
| if main_content: | |
| return self.normalize_text(main_content.get_text(strip=True)) | |
| return self.normalize_text(soup.get_text(strip=True)) | |
| def get_page_metadata(self, soup, url): | |
| """Extract metadata from the page""" | |
| metadata = { | |
| 'title': None, | |
| 'description': None, | |
| 'importance': 0, | |
| 'category': 'Optional' | |
| } | |
| # Title extraction with normalization | |
| title = ( | |
| soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else | |
| soup.find('title').text if soup.find('title') else | |
| soup.find('h1').text if soup.find('h1') else | |
| url.split('/')[-1] | |
| ) | |
| metadata['title'] = self.normalize_text(title) | |
| # Description extraction with normalization | |
| description = ( | |
| soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else | |
| soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else | |
| "" | |
| ) | |
| metadata['description'] = self.normalize_text(description) | |
| # Calculate importance based on various factors | |
| importance = 0 | |
| if 'docs' in url.lower() or 'documentation' in url.lower(): | |
| importance += 5 | |
| metadata['category'] = 'Docs' | |
| if 'api' in url.lower(): | |
| importance += 4 | |
| metadata['category'] = 'API' | |
| if 'guide' in url.lower() or 'tutorial' in url.lower(): | |
| importance += 3 | |
| metadata['category'] = 'Guides' | |
| if 'example' in url.lower(): | |
| importance += 2 | |
| metadata['category'] = 'Examples' | |
| if 'blog' in url.lower(): | |
| importance += 1 | |
| metadata['category'] = 'Blog' | |
| metadata['importance'] = importance | |
| return metadata | |
| async def crawl_page(self, url, depth, base_domain): | |
| """Crawl a single page and extract information""" | |
| if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages: | |
| return [] | |
| try: | |
| response = requests.get(url, headers=self.headers, timeout=self.timeout) | |
| response.encoding = 'utf-8' # Explicitly set encoding | |
| response.raise_for_status() | |
| self.visited_urls.add(url) | |
| soup = BeautifulSoup(response.text, 'html.parser') | |
| content = self.extract_content(soup) | |
| metadata = self.get_page_metadata(soup, url) | |
| self.url_content[url] = content | |
| self.url_metadata[url] = metadata | |
| # Find all links | |
| links = [] | |
| for a in soup.find_all('a', href=True): | |
| next_url = urljoin(url, a['href']) | |
| if self.is_valid_url(next_url, base_domain): | |
| links.append(next_url) | |
| return links | |
| except Exception as e: | |
| logger.error(f"Error crawling {url}: {str(e)}") | |
| return [] | |
| async def crawl_website(self, start_url): | |
| """Crawl website starting from the given URL""" | |
| base_domain = start_url | |
| queue = [(start_url, 0)] | |
| seen = {start_url} | |
| while queue and len(self.visited_urls) < self.max_pages: | |
| current_url, depth = queue.pop(0) | |
| if depth > self.max_depth: | |
| continue | |
| links = await self.crawl_page(current_url, depth, base_domain) | |
| for link in links: | |
| if link not in seen: | |
| seen.add(link) | |
| queue.append((link, depth + 1)) | |
| def generate_llms_txt(self): | |
| """Generate llms.txt content from crawled data""" | |
| # Sort URLs by importance | |
| sorted_urls = sorted( | |
| self.url_metadata.items(), | |
| key=lambda x: (x[1]['importance'], x[0]), | |
| reverse=True | |
| ) | |
| # Group URLs by category | |
| categorized_urls = defaultdict(list) | |
| for url, metadata in sorted_urls: | |
| categorized_urls[metadata['category']].append((url, metadata)) | |
| # Generate content | |
| content = [] | |
| # Add main title and description | |
| if sorted_urls: | |
| main_metadata = sorted_urls[0][1] | |
| content.append(f"# {main_metadata['title']}\n") | |
| content.append(f"> {main_metadata['description']}\n") | |
| # Add categorized sections | |
| priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional'] | |
| for category in priority_order: | |
| if category in categorized_urls: | |
| content.append(f"\n## {category}\n") | |
| for url, metadata in categorized_urls[category]: | |
| title = metadata['title'] | |
| desc = metadata['description'] | |
| if desc: | |
| content.append(f"- [{title}]({url}): {desc[:100]}...\n") | |
| else: | |
| content.append(f"- [{title}]({url})\n") | |
| return "\n".join(content) | |
| async def process_url(url, max_depth, max_pages): | |
| """Process URL and generate llms.txt""" | |
| try: | |
| # Add https:// if not present | |
| if not url.startswith(('http://', 'https://')): | |
| url = 'https://' + url | |
| # Validate URL format | |
| try: | |
| result = urlparse(url) | |
| if not all([result.scheme, result.netloc]): | |
| return "", "Invalid URL format. Please enter a valid URL." | |
| except: | |
| return "", "Invalid URL format. Please enter a valid URL." | |
| # Create crawler and process | |
| crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages)) | |
| await crawler.crawl_website(url) | |
| content = crawler.generate_llms_txt() | |
| return content, f"Successfully crawled {len(crawler.visited_urls)} pages. You can now copy the generated content." | |
| except Exception as e: | |
| return "", f"Error: {str(e)}" | |
| # Create the Gradio interface with custom CSS for Open Sans font | |
| css = """ | |
| @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap'); | |
| body, .gradio-container { | |
| font-family: 'Open Sans', sans-serif !important; | |
| } | |
| .gr-box { | |
| border-radius: 8px !important; | |
| border: 1px solid #e5e7eb !important; | |
| } | |
| .gr-button { | |
| font-family: 'Open Sans', sans-serif !important; | |
| font-weight: 600 !important; | |
| } | |
| .gr-input { | |
| font-family: 'Open Sans', sans-serif !important; | |
| } | |
| """ | |
| # Create the Gradio interface | |
| iface = gr.Interface( | |
| fn=lambda url, max_depth, max_pages: asyncio.run(process_url(url, max_depth, max_pages)), | |
| inputs=[ | |
| gr.Textbox( | |
| label="Website URL", | |
| placeholder="Enter the website URL (e.g., example.com or https://example.com)", | |
| info="The URL will be automatically prefixed with https:// if no protocol is specified." | |
| ), | |
| gr.Slider( | |
| minimum=1, | |
| maximum=5, | |
| value=3, | |
| step=1, | |
| label="Maximum Crawl Depth", | |
| info="Higher values will result in more thorough but slower crawling" | |
| ), | |
| gr.Slider( | |
| minimum=10, | |
| maximum=100, | |
| value=50, | |
| step=10, | |
| label="Maximum Pages to Crawl", | |
| info="Higher values will result in more comprehensive but slower results" | |
| ) | |
| ], | |
| outputs=[ | |
| gr.Textbox( | |
| label="Generated llms.txt Content", | |
| lines=20, | |
| info="Copy this content to create your llms.txt file" | |
| ), | |
| gr.Textbox(label="Status") | |
| ], | |
| title="llms.txt Generator", | |
| description="Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.", | |
| theme=gr.themes.Soft(), | |
| css=css | |
| ) | |
| # Launch the app | |
| if __name__ == "__main__": | |
| iface.launch() |