Update app.py

app.py

@@ -9,6 +9,7 @@ from collections import defaultdict
 import unicodedata
 import logging
 import ssl
+import brotli  # Add this import
 
 logging.basicConfig(level=logging.INFO)
 logger = logging.getLogger(__name__)
@@ -28,377 +29,164 @@ class WebsiteCrawler:
             "Accept-Encoding": "gzip, deflate, br",
             "DNT": "1",
             "Connection": "keep-alive",
-            "Upgrade-Insecure-Requests": "1"
+            "Upgrade-Insecure-Requests": "1",
         }
+        self.session = None
 
-    def determine_category_importance(self, url, title, desc):
-        """Improved category detection"""
-        url_lower = url.lower()
-        path = urlparse(url).path.lower()
-        
-        # Homepage
-        if path == "/" or path == "":
-            return "Main", 10
-            
-        # Documentation and Help
-        if any(x in url_lower for x in ['/docs', '/documentation', '/faq', '/help', 'frequently-asked-questions']):
-            return "Documentation", 8
-            
-        # API and Developer
-        elif any(x in url_lower for x in ['/api', '/developer', 'developers']):
-            return "API", 8
-            
-        # About/Company pages
-        elif any(x in url_lower for x in [
-            '/about', '/company', '/references', '/work-with-us',
-            'careers', '/team', '/contact', '/about-us'
-        ]):
-            return "About", 7
-            
-        # News and Events    
-        elif any(x in url_lower for x in [
-            '/news', '/blog', '/events', '/press',
-            'research', 'power-of', 'latest'
-        ]):
-            return "News", 5
-            
-        # Tools and Services
-        elif any(x in url_lower for x in [
-            '/tools', '/quote', '/pricing', '/services',
-            '/translate', '/order', '/buy'
-        ]):
-            return "Tools", 6
-        
-        # Check if URL path contains non-ASCII or percent-encoded characters
-        if bool(re.search(r'[^\x00-\x7F]', path)) or bool(re.search(r'%[0-9A-F]{2}', path)):
-            return "Optional", 0
-            
-        return "Optional", 1
-
-    def is_duplicate_content(self, desc, title, url):
-        """Improved duplicate/translation detection"""
-        if not desc or not title:
-            return False
-                
-        # Skip non-latin character URLs or URLs with percent-encoded non-ASCII
-        if bool(re.search(r'[^\x00-\x7F]', url)) or bool(re.search(r'%[0-9A-F]{2}', url)):
-            return True
-    
-                
-        # Skip common translation paths
-        translation_indicators = [
-            '/welcome', '/bienvenue', '/willkommen', '/benvenuto', 
-            '/tervetuloa', '/bienvenido', '/velkommen', '/welkom',
-            'translate.com/', '/translate/', '/translation/'
-        ]
-        if any(indicator in url.lower() for indicator in translation_indicators):
-            url_path = urlparse(url).path.lower()
-            if url_path != '/':  # Don't skip homepage
-                return True
-                
-        # Check for similar content length and patterns
-        for existing_metadata in self.url_metadata.values():
-            existing_desc = existing_metadata.get("description", "")
-            existing_title = existing_metadata.get("title", "")
-            if not existing_desc or not existing_title:
-                continue
-                    
-            # If descriptions are very similar in length, likely a translation
-            if (abs(len(desc) - len(existing_desc)) < 20 and
-                len(desc) > 50 and
-                desc != existing_desc):  # Allow exact duplicates for main page
-                return True
-                    
-        return False
+    async def get_session(self):
+        if self.session is None:
+            ssl_context = ssl.create_default_context()
+            ssl_context.check_hostname = False
+            ssl_context.verify_mode = ssl.CERT_NONE
+
+            # Configure client with brotli support
+            connector = aiohttp.TCPConnector(ssl=ssl_context)
+            self.session = aiohttp.ClientSession(
+                connector=connector, timeout=aiohttp.ClientTimeout(total=30)
+            )
+        return self.session
+
+    async def decode_response(self, response):
+        """Handle various content encodings including brotli"""
+        content_encoding = response.headers.get("Content-Encoding", "").lower()
+        content = await response.read()
+
+        if content_encoding == "br":
+            try:
+                decoded = brotli.decompress(content)
+                return decoded.decode("utf-8", errors="ignore")
+            except Exception as e:
+                logger.error(f"Error decoding brotli content: {str(e)}")
+                return content.decode("utf-8", errors="ignore")
+        elif content_encoding == "gzip":
+            import gzip
+
+            try:
+                decoded = gzip.decompress(content)
+                return decoded.decode("utf-8", errors="ignore")
+            except Exception as e:
+                logger.error(f"Error decoding gzip content: {str(e)}")
+                return content.decode("utf-8", errors="ignore")
+        else:
+            return content.decode("utf-8", errors="ignore")
 
     def clean_text(self, text, is_title=False):
-        """Improved text cleaning"""
-        if not text or len(text.strip()) < 2:
+        """Clean and normalize text"""
+        if not text:
             return ""
-            
         # Normalize unicode characters
         text = unicodedata.normalize("NFKD", text)
         text = re.sub(r"[^\x00-\x7F]+", "", text)
-        
-        # Remove any template variables/placeholders
-        text = re.sub(r'\{\{.*?\}\}', '', text)
-        text = re.sub(r'\{\%.*?\%\}', '', text)
-        text = re.sub(r'\${.*?\}', '', text)
-        
+
         if is_title:
             # Remove common suffixes and fragments for titles
-            text = re.sub(r'^\s*Welcome to\s+', '', text)
-            text = re.sub(r'\s*[\|\-#:•].*', '', text)
-            text = re.sub(r'\s+Homepage$', '', text, flags=re.IGNORECASE)
-            
-            # Handle overly generic titles
-            if text.lower() in ['features', 'home', 'homepage', 'welcome']:
-                return ""
-        
-        # Only return if we have meaningful text
-        cleaned = " ".join(text.split()).strip()
-        if len(cleaned.split()) < 2 and not is_title:  # Allow single-word titles
-            return ""
-            
-        return cleaned
+            text = re.sub(r"\s*[\|\-#:•].*", "", text)
+            text = re.sub(r"^\s*Welcome to\s+", "", text)
+            text = text.replace("docusaurus_skipToContent_fallback", "")
+
+        return " ".join(text.split()).strip()
 
-    
-    def clean_description(self, desc):
-        """Clean description text"""
-        if not desc:
-            return ""
-        # Remove leading dashes, hyphens, or colons
-        desc = re.sub(r"^[-:\s]+", "", desc)
-        # Remove any strings that are just "Editors", "APIs", etc.
-        if len(desc.split()) <= 1:
-            return ""
-        return desc.strip()
-
-
-    def extract_homepage_description(self, soup):
-        """Extract description from homepage with multiple fallbacks"""
-        # Try meta description first
-        meta_desc = soup.find("meta", {"name": "description"})
-        if meta_desc and meta_desc.get("content"):
-            desc = meta_desc["content"]
-            if desc and len(desc.strip()) > 20:
-                return self.clean_text(desc)
-
-        # Try OpenGraph description
-        og_desc = soup.find("meta", property="og:description")
-        if og_desc and og_desc.get("content"):
-            desc = og_desc["content"]
-            if desc and len(desc.strip()) > 20:
-                return self.clean_text(desc)
-
-        # Try first significant paragraph
-        for p in soup.find_all("p"):
-            text = p.get_text().strip()
-            if len(text) > 50 and not any(x in text.lower() for x in ["cookie", "accept", "privacy"]):
-                return self.clean_text(text)
-
-        # Try main content area if exists
-        main = soup.find("main")
-        if main:
-            first_p = main.find("p")
-            if first_p:
-                text = first_p.get_text().strip()
-                if len(text) > 50:
-                    return self.clean_text(text)
-
-        return None
-
-    async def crawl_page(self, url, depth, base_domain):
-        """Crawl a single page and extract information"""
-        if (
-            depth > self.max_depth
-            or url in self.visited_urls
-            or len(self.visited_urls) >= self.max_pages
-        ):
-            return []
-    
-        try:
-            await asyncio.sleep(1)  # Be polite to servers
-            async with aiohttp.ClientSession() as session:
-                async with session.get(url, headers=self.headers, allow_redirects=True) as response:
-                    if response.status == 403:
-                        # Try with alternative headers
-                        alt_headers = {
-                            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
-                            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                        }
-                        async with session.get(url, headers=alt_headers, allow_redirects=True) as retry_response:
-                            if retry_response.status != 200:
-                                return []
-                            text = await retry_response.text()
-                    elif response.status != 200:
-                        return []
-                    else:
-                        text = await response.text()
-    
-                    self.visited_urls.add(url)
-                    soup = BeautifulSoup(text, "html.parser")    
-                    
-                    # Extract title with fallbacks
-                    title = None
-                    meta_title = soup.find("meta", property="og:title")
-                    if meta_title and meta_title.get("content"):
-                        title = meta_title["content"]
-                    if not title:
-                        title_tag = soup.find("title")
-                        if title_tag:
-                            title = title_tag.text
-                    if not title:
-                        h1_tag = soup.find("h1")
-                        if h1_tag:
-                            title = h1_tag.text
-                    if not title:
-                        title = url.split("/")[-1]
-    
-                    title = self.clean_text(title, is_title=True)
-    
-                    # Extract description with fallbacks
-                    desc = None
-                    meta_desc = soup.find("meta", {"name": "description"})
-                    if meta_desc and meta_desc.get("content"):
-                        desc = meta_desc["content"]
-                    if not desc:
-                        og_desc = soup.find("meta", property="og:description")
-                        if og_desc and og_desc.get("content"):
-                            desc = og_desc["content"]
-                    if not desc:
-                        first_p = soup.find("p")
-                        if first_p:
-                            desc = first_p.text
-    
-                    desc = self.clean_text(desc) if desc else ""
-    
-                    # Skip if it's duplicate content
-                    if self.is_duplicate_content(desc, title, url):
-                        return []
-    
-                    # Determine category and importance
-                    category, importance = self.determine_category_importance(url, title, desc)
-    
-                    # Store metadata
-                    clean_url = re.sub(r"#.*", "", url).rstrip("/")
-                    if title and len(title.strip()) > 0:  # Only store if we have a valid title
-                        logger.info(f"Storing metadata for {clean_url}: {title[:30]}...")
-                        self.url_metadata[clean_url] = {
-                            "title": title,
-                            "description": desc,
-                            "category": category,
-                            "importance": importance,
-                        }
-    
-                    # Find links
-                    links = []
-                    for a in soup.find_all("a", href=True):
-                        href = a["href"]
-                        if not any(
-                            x in href.lower()
-                            for x in ["javascript:", "mailto:", ".pdf", ".jpg", ".png", ".gif"]
-                        ):
-                            next_url = urljoin(url, href)
-                            if urlparse(next_url).netloc == base_domain:
-                                links.append(next_url)
-                    return links
-    
-        except Exception as e:
-            logger.error(f"Error crawling {url}: {str(e)}")
-            return []
-                
     async def process_homepage(self, url):
         """Specifically process the homepage to extract key metadata"""
         try:
-            # Configure SSL context
-            ssl_context = ssl.create_default_context()
-            ssl_context.check_hostname = False
-            ssl_context.verify_mode = ssl.CERT_NONE
-            
-            connector = aiohttp.TCPConnector(ssl=ssl_context)
-            timeout = aiohttp.ClientTimeout(total=30)
-            
-            async with aiohttp.ClientSession(connector=connector, timeout=timeout) as session:
-                async with session.get(url, headers=self.headers, allow_redirects=True) as response:
-                    if response.status != 200:
-                        raise Exception(f"Failed to fetch homepage: status {response.status}")
-                        
-                    try:
-                        text = await response.text()
-                    except UnicodeDecodeError:
-                        text = await response.read()
-                        text = text.decode('utf-8', errors='ignore')
-                    
-                    soup = BeautifulSoup(text, "html.parser")
-                    
-                    # Extract site name with more fallbacks
-                    site_name = None
-                    # Try meta tags first
-                    site_meta = soup.find("meta", property="og:site_name")
-                    if site_meta and site_meta.get("content"):
-                        site_name = site_meta["content"]
-                        
-                    # Try structured data
-                    if not site_name:
-                        schema = soup.find("script", type="application/ld+json")
-                        if schema:
-                            try:
-                                import json
-                                data = json.loads(schema.string)
-                                if isinstance(data, dict):
-                                    site_name = data.get("name") or data.get("organizationName")
-                            except:
-                                pass
-                                
-                    # Try title tag
-                    if not site_name:
-                        title_tag = soup.find("title")
-                        if title_tag:
-                            site_name = title_tag.text.split('|')[0].strip()
-                            
-                    # Last resort - use domain name
-                    if not site_name:
-                        site_name = urlparse(url).netloc.split('.')[0].capitalize()
-    
-                    # Get homepage description
-                    description = self.extract_homepage_description(soup)
-    
-                    self.homepage_metadata = {
-                        "site_name": self.clean_text(site_name, is_title=True),
-                        "description": description
-                    }
-    
+            session = await self.get_session()
+            async with session.get(
+                url, headers=self.headers, allow_redirects=True
+            ) as response:
+                if response.status != 200:
+                    raise Exception(
+                        f"Failed to fetch homepage: status {response.status}"
+                    )
+
+                text = await self.decode_response(response)
+                soup = BeautifulSoup(text, "html.parser")
+
+                # Extract site name
+                site_name = None
+                site_meta = soup.find("meta", property="og:site_name")
+                if site_meta and site_meta.get("content"):
+                    site_name = site_meta["content"]
+
+                if not site_name:
+                    title_tag = soup.find("title")
+                    if title_tag:
+                        site_name = title_tag.text.split("|")[0].strip()
+
+                if not site_name:
+                    site_name = urlparse(url).netloc.split(".")[0].capitalize()
+
+                # Get homepage description
+                description = None
+                meta_desc = soup.find("meta", {"name": "description"})
+                if meta_desc and meta_desc.get("content"):
+                    description = meta_desc["content"]
+
+                if not description:
+                    og_desc = soup.find("meta", property="og:description")
+                    if og_desc and og_desc.get("content"):
+                        description = og_desc["content"]
+
+                if not description:
+                    first_p = soup.find("p")
+                    if first_p:
+                        description = first_p.text
+
+                self.homepage_metadata = {
+                    "site_name": self.clean_text(site_name, is_title=True),
+                    "description": (
+                        self.clean_text(description) if description else None
+                    ),
+                }
+
         except Exception as e:
             logger.error(f"Error processing homepage {url}: {str(e)}")
             self.homepage_metadata = {
-                "site_name": urlparse(url).netloc.split('.')[0].capitalize(),
-                "description": None
+                "site_name": urlparse(url).netloc.split(".")[0].capitalize(),
+                "description": None,
             }
-    
+
     async def crawl_website(self, start_url):
         """Crawl website starting from the given URL"""
         try:
             # First process the homepage
             logger.info(f"Processing homepage: {start_url}")
             await self.process_homepage(start_url)
-            
+
             base_domain = urlparse(start_url).netloc
             queue = [(start_url, 0)]
             seen = {start_url}
-    
+
             while queue and len(self.visited_urls) < self.max_pages:
                 current_url, depth = queue.pop(0)
                 if depth > self.max_depth:
                     continue
-    
+
                 logger.info(f"Crawling page: {current_url} (depth: {depth})")
                 links = await self.crawl_page(current_url, depth, base_domain)
                 logger.info(f"Found {len(links)} links on {current_url}")
-                
+
                 for link in links:
                     if link not in seen and urlparse(link).netloc == base_domain:
                         seen.add(link)
                         queue.append((link, depth + 1))
-    
+
             logger.info(f"Crawl completed. Visited {len(self.visited_urls)} pages")
-            
+
         except Exception as e:
             logger.error(f"Error during crawl: {str(e)}")
             raise
+        finally:
+            await self.cleanup()
 
     def generate_llms_txt(self):
         """Generate llms.txt content"""
-        logger.info(f"Starting generate_llms_txt with {len(self.url_metadata)} URLs")
-        
         if not self.url_metadata:
-            logger.error("No URL metadata found")
             return "No content was found to generate llms.txt"
-    
+
         # Sort URLs by importance and remove duplicates
         sorted_urls = []
         seen_titles = set()
-    
+
         for url, metadata in sorted(
             self.url_metadata.items(),
             key=lambda x: (x[1]["importance"], x[0]),
@@ -407,95 +195,55 @@ class WebsiteCrawler:
             if metadata["title"] not in seen_titles:
                 sorted_urls.append((url, metadata))
                 seen_titles.add(metadata["title"])
-    
-        logger.info(f"Found {len(sorted_urls)} unique URLs after deduplication")
-        
+
         if not sorted_urls:
-            logger.error("No valid URLs found after sorting")
             return "No valid content was found"
-    
+
         # Generate content
         content = []
-    
+
         # Use homepage metadata for main title and description
         main_title = self.homepage_metadata.get("site_name", "Welcome")
         homepage_description = self.homepage_metadata.get("description")
-    
-        logger.info(f"Homepage title: {main_title}")
-        logger.info(f"Homepage description: {homepage_description}")
-    
+
         content.append(f"# {main_title}")
         if homepage_description:
             content.append(f"\n> {homepage_description}")
-        elif len(sorted_urls) > 0:
-            # Fallback to first good description from content if no homepage description
+        else:
+            # Fallback to first good description from content
             for _, metadata in sorted_urls:
                 desc = self.clean_description(metadata["description"])
                 if desc and len(desc) > 20 and "null" not in desc.lower():
                     content.append(f"\n> {desc}")
                     break
-    
+
         # Group by category
         categories = defaultdict(list)
         for url, metadata in sorted_urls:
             if metadata["title"] and url:
                 categories[metadata["category"]].append((url, metadata))
-    
-        logger.info(f"Categories found: {list(categories.keys())}")
-    
-        # Add sections in a logical order
-        category_order = [
-            "Main",
-            "Documentation",
-            "API",
-            "Tools",
-            "About",
-            "News",
-            "Optional"
-        ]
-    
-        # Only show Main section if it has content different from the homepage description
-        if "Main" in categories:
-            main_content = categories["Main"]
-            if len(main_content) == 1 and main_content[0][1]["description"] == homepage_description:
-                logger.info("Removing duplicate Main content")
-                del categories["Main"]
-    
-        for category in category_order:
-            if category in categories and categories[category]:
-                logger.info(f"Processing category {category} with {len(categories[category])} items")
+
+        # Add sections
+        for category in ["Docs", "API", "Guides", "Examples", "Blog", "Optional"]:
+            if category in categories:
                 content.append(f"\n## {category}")
-                
-                # Sort links within category by importance and description length
-                category_links = sorted(
-                    categories[category],
-                    key=lambda x: (-len(x[1]["description"] or ""), x[1]["title"])
-                )
-    
+
+                # Add links without extra newlines
                 links = []
-                seen_desc = set()  # Avoid duplicate descriptions within category
-                for url, metadata in category_links:
+                for url, metadata in categories[category]:
                     title = metadata["title"].strip()
                     desc = self.clean_description(metadata["description"])
-                    
-                    # Skip if description is duplicate within category
-                    if desc in seen_desc:
-                        continue
-                    seen_desc.add(desc)
-                    
                     if desc:
                         links.append(f"- [{title}]({url}): {desc}")
                     else:
                         links.append(f"- [{title}]({url})")
-    
+
                 content.append("\n".join(links))
-    
-        final_content = "\n".join(content)
-        logger.info(f"Generated content length: {len(final_content)}")
-        return final_content
-        
+
+        return "\n".join(content)
 
 
+# Process URL function (outside the class)
 async def process_url(url, max_depth, max_pages):
     """Process URL and generate llms.txt"""
     try:
@@ -509,11 +257,11 @@ async def process_url(url, max_depth, max_pages):
             return "", "Invalid URL format. Please enter a valid URL."
 
         logger.info(f"Starting crawl of {url}")
-        
+
         # Process website
         crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
         await crawler.crawl_website(url)
-        
+
         logger.info("Generating llms.txt content")
         content = crawler.generate_llms_txt()
 
@@ -570,6 +318,8 @@ with gr.Blocks(
     }
 """,
 ) as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification.")
 
     with gr.Row():
         url_input = gr.Textbox(
@@ -606,4 +356,4 @@ with gr.Blocks(
     )
 
 if __name__ == "__main__":
-    iface.launch()
+    iface.launch()