Update app.py

app.py

@@ -43,30 +43,2003 @@ class WebsiteCrawler:
         text = ' '.join(text.split())
         return text
 
+    def clean_url(self, url):
+        """Clean URL by removing fragments and unnecessary parameters"""
+        # Remove fragments (everything after #)
+        url = re.sub(r'#.*
+
+    def is_valid_url(self, url, base_domain):
+        """Check if URL is valid and belongs to the same domain"""
+        try:
+            parsed = urlparse(url)
+            base_parsed = urlparse(base_domain)
+            return (parsed.netloc == base_parsed.netloc and
+                   parsed.scheme in ['http', 'https'] and
+                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
+        except:
+            return False
+
+    def extract_content(self, soup):
+        """Extract meaningful content from HTML"""
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
+            element.decompose()
+
+        # Get main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
+        if main_content:
+            return self.normalize_text(main_content.get_text(strip=True))
+        return self.normalize_text(soup.get_text(strip=True))
+
+    def get_page_metadata(self, soup, url):
+        """Extract metadata from the page"""
+        metadata = {
+            'title': None,
+            'description': None,
+            'importance': 0,
+            'category': 'Optional'
+        }
+
+        # Title extraction with cleaning
+        title = (
+            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
+            soup.find('title').text if soup.find('title') else
+            soup.find('h1').text if soup.find('h1') else
+            url.split('/')[-1]
+        )
+        metadata['title'] = self.clean_title(title)
+
+        # Description extraction with cleaning
+        description = (
+            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
+            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
+            ""
+        )
+        metadata['description'] = self.clean_description(description)
+
+        # Calculate importance and category
+        url_lower = url.lower()
+        if 'docs' in url_lower or 'documentation' in url_lower:
+            metadata['importance'] = 5
+            metadata['category'] = 'Docs'
+        elif 'api' in url_lower:
+            metadata['importance'] = 4
+            metadata['category'] = 'API'
+        elif 'guide' in url_lower or 'tutorial' in url_lower:
+            metadata['importance'] = 3
+            metadata['category'] = 'Guides'
+        elif 'example' in url_lower:
+            metadata['importance'] = 2
+            metadata['category'] = 'Examples'
+        elif 'blog' in url_lower:
+            metadata['importance'] = 1
+            metadata['category'] = 'Blog'
+
+        return metadata
+
+    async def crawl_page(self, url, depth, base_domain):
+        """Crawl a single page and extract information"""
+        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
+            return []
+
+        try:
+            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response.encoding = 'utf-8'
+            response.raise_for_status()
+            self.visited_urls.add(url)
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            content = self.extract_content(soup)
+            metadata = self.get_page_metadata(soup, url)
+            
+            self.url_content[url] = content
+            self.url_metadata[url] = metadata
+
+            # Find all links
+            links = []
+            for a in soup.find_all('a', href=True):
+                next_url = urljoin(url, a['href'])
+                if self.is_valid_url(next_url, base_domain):
+                    links.append(next_url)
+
+            return links
+
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            return []
+
+    async def crawl_website(self, start_url):
+        """Crawl website starting from the given URL"""
+        base_domain = start_url
+        queue = [(start_url, 0)]
+        seen = {start_url}
+
+        while queue and len(self.visited_urls) < self.max_pages:
+            current_url, depth = queue.pop(0)
+            
+            if depth > self.max_depth:
+                continue
+
+            links = await self.crawl_page(current_url, depth, base_domain)
+            
+            for link in links:
+                if link not in seen:
+                    seen.add(link)
+                    queue.append((link, depth + 1))
+
+    def generate_llms_txt(self):
+        """Generate llms.txt content from crawled data"""
+        # Clean and deduplicate metadata
+        cleaned_metadata = self.remove_duplicate_content(self.url_metadata)
+        
+        # Sort URLs by importance
+        sorted_urls = sorted(
+            cleaned_metadata.items(),
+            key=lambda x: (x[1]['importance'], x[0]),
+            reverse=True
+        )
+
+        if not sorted_urls:
+            return "No content was found to generate llms.txt"
+
+        # Group URLs by category
+        categorized_urls = defaultdict(list)
+        for url, metadata in sorted_urls:
+            categorized_urls[metadata['category']].append((url, metadata))
+
+        # Generate content
+        content = []
+        
+        # Add main title and description
+        main_metadata = sorted_urls[0][1]
+        content.append(f"# {main_metadata['title']}")
+        if main_metadata['description']:
+            content.append(f"\n> {main_metadata['description']}")
+        
+        # Add categorized sections
+        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
+        
+        for category in priority_order:
+            if category in categorized_urls:
+                content.append(f"\n## {category}")
+                for url, metadata in categorized_urls[category]:
+                    if metadata['description']:
+                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
+                    else:
+                        content.append(f"\n- [{metadata['title']}]({url})")
+
+        return "\n".join(content)
+
+async def process_url(url, max_depth, max_pages):
+    """Process URL and generate llms.txt"""
+    try:
+        # Add https:// if not present
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        # Validate URL format
+        try:
+            result = urlparse(url)
+            if not all([result.scheme, result.netloc]):
+                return "", "Invalid URL format. Please enter a valid URL."
+        except:
+            return "", "Invalid URL format. Please enter a valid URL."
+
+        # Create crawler and process
+        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
+        await crawler.crawl_website(url)
+        content = crawler.generate_llms_txt()
+        
+        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
+    
+    except Exception as e:
+        return "", f"Error: {str(e)}"
+
+# Create custom theme
+theme = gr.themes.Soft(
+    primary_hue="blue",
+    font="Open Sans"
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=theme,     css="""
+    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
+    
+    .gradio-container {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+    
+    .gr-button {
+        font-family: 'Open Sans', sans-serif !important;
+        font-weight: 600 !important;
+    }
+
+    /* Primary color customization */
+    .primary-btn {
+        background-color: #2436d4 !important;
+        color: white !important;
+    }
+
+    .primary-btn:hover {
+        background-color: #1c2aa8 !important;
+    }
+    
+    [data-testid="textbox"] {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+""") as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Website URL", 
+            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
+            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            depth_input = gr.Slider(
+                minimum=1, 
+                maximum=5, 
+                value=3, 
+                step=1, 
+                label="Maximum Crawl Depth",
+                info="Higher values will result in more thorough but slower crawling"
+            )
+        with gr.Column():
+            pages_input = gr.Slider(
+                minimum=10, 
+                maximum=100, 
+                value=50, 
+                step=10, 
+                label="Maximum Pages to Crawl",
+                info="Higher values will result in more comprehensive but slower results"
+            )
+    
+    generate_btn = gr.Button("Generate llms.txt", variant="primary")
+    
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
+            lines=20,
+            max_lines=30,
+            show_copy_button=True,
+            container=True,
+            scale=2,
+            interactive=True
+        )
+    
+    status = gr.Textbox(label="Status")
+    
+    generate_btn.click(
+        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
+        inputs=[url_input, depth_input, pages_input],
+        outputs=[output, status]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    iface.launch()
+, '', url)
+        # Remove trailing slashes
+        url = url.rstrip('/')
+        return url
+
+    def remove_duplicate_content(self, urls_metadata):
+        """Remove duplicate content based on similar titles and URLs"""
+        seen_content = {}
+        cleaned_metadata = {}
+        
+        for url, metadata in urls_metadata.items():
+            clean_url = self.clean_url(url)
+            base_url = clean_url.split('#')[0]  # Remove hash fragments
+            
+            # Create a content signature based on title and base URL
+            title = metadata['title'].lower()
+            
+            # Skip entries that are just fragments of the same page
+            if base_url in seen_content:
+                # Keep the one with the shortest URL (usually the main page)
+                if len(clean_url) < len(seen_content[base_url]):
+                    cleaned_metadata[clean_url] = metadata
+                    cleaned_metadata.pop(seen_content[base_url], None)
+                    seen_content[base_url] = clean_url
+                continue
+            
+            seen_content[base_url] = clean_url
+            cleaned_metadata[clean_url] = metadata
+            
+        return cleaned_metadata
+
     def clean_title(self, title):
         """Clean and format titles"""
+        if not title:
+            return ""
+        
         title = self.normalize_text(title)
-        # Remove common suffixes
-        title = re.sub(r'\s*\|\s*.*$', '', title)  # Remove pipe and everything after
-        title = re.sub(r'\s*-\s*.*$', '', title)   # Remove dash and everything after
-        title = title.strip()
-        return title
+        
+        # Remove common suffixes and prefixes
+        patterns = [
+            r'\s*\|\s*.*
+
+    def is_valid_url(self, url, base_domain):
+        """Check if URL is valid and belongs to the same domain"""
+        try:
+            parsed = urlparse(url)
+            base_parsed = urlparse(base_domain)
+            return (parsed.netloc == base_parsed.netloc and
+                   parsed.scheme in ['http', 'https'] and
+                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
+        except:
+            return False
+
+    def extract_content(self, soup):
+        """Extract meaningful content from HTML"""
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
+            element.decompose()
+
+        # Get main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
+        if main_content:
+            return self.normalize_text(main_content.get_text(strip=True))
+        return self.normalize_text(soup.get_text(strip=True))
+
+    def get_page_metadata(self, soup, url):
+        """Extract metadata from the page"""
+        metadata = {
+            'title': None,
+            'description': None,
+            'importance': 0,
+            'category': 'Optional'
+        }
+
+        # Title extraction with cleaning
+        title = (
+            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
+            soup.find('title').text if soup.find('title') else
+            soup.find('h1').text if soup.find('h1') else
+            url.split('/')[-1]
+        )
+        metadata['title'] = self.clean_title(title)
+
+        # Description extraction with cleaning
+        description = (
+            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
+            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
+            ""
+        )
+        metadata['description'] = self.clean_description(description)
+
+        # Calculate importance and category
+        url_lower = url.lower()
+        if 'docs' in url_lower or 'documentation' in url_lower:
+            metadata['importance'] = 5
+            metadata['category'] = 'Docs'
+        elif 'api' in url_lower:
+            metadata['importance'] = 4
+            metadata['category'] = 'API'
+        elif 'guide' in url_lower or 'tutorial' in url_lower:
+            metadata['importance'] = 3
+            metadata['category'] = 'Guides'
+        elif 'example' in url_lower:
+            metadata['importance'] = 2
+            metadata['category'] = 'Examples'
+        elif 'blog' in url_lower:
+            metadata['importance'] = 1
+            metadata['category'] = 'Blog'
+
+        return metadata
+
+    async def crawl_page(self, url, depth, base_domain):
+        """Crawl a single page and extract information"""
+        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
+            return []
+
+        try:
+            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response.encoding = 'utf-8'
+            response.raise_for_status()
+            self.visited_urls.add(url)
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            content = self.extract_content(soup)
+            metadata = self.get_page_metadata(soup, url)
+            
+            self.url_content[url] = content
+            self.url_metadata[url] = metadata
+
+            # Find all links
+            links = []
+            for a in soup.find_all('a', href=True):
+                next_url = urljoin(url, a['href'])
+                if self.is_valid_url(next_url, base_domain):
+                    links.append(next_url)
+
+            return links
+
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            return []
+
+    async def crawl_website(self, start_url):
+        """Crawl website starting from the given URL"""
+        base_domain = start_url
+        queue = [(start_url, 0)]
+        seen = {start_url}
+
+        while queue and len(self.visited_urls) < self.max_pages:
+            current_url, depth = queue.pop(0)
+            
+            if depth > self.max_depth:
+                continue
+
+            links = await self.crawl_page(current_url, depth, base_domain)
+            
+            for link in links:
+                if link not in seen:
+                    seen.add(link)
+                    queue.append((link, depth + 1))
+
+    def generate_llms_txt(self):
+        """Generate llms.txt content from crawled data"""
+        # Sort URLs by importance
+        sorted_urls = sorted(
+            self.url_metadata.items(),
+            key=lambda x: (x[1]['importance'], x[0]),
+            reverse=True
+        )
+
+        if not sorted_urls:
+            return "No content was found to generate llms.txt"
+
+        # Group URLs by category
+        categorized_urls = defaultdict(list)
+        for url, metadata in sorted_urls:
+            categorized_urls[metadata['category']].append((url, metadata))
+
+        # Generate content
+        content = []
+        
+        # Add main title and description
+        main_metadata = sorted_urls[0][1]
+        content.append(f"# {main_metadata['title']}")
+        if main_metadata['description']:
+            content.append(f"\n> {main_metadata['description']}")
+        
+        # Add categorized sections
+        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
+        
+        for category in priority_order:
+            if category in categorized_urls:
+                content.append(f"\n## {category}")
+                for url, metadata in categorized_urls[category]:
+                    if metadata['description']:
+                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
+                    else:
+                        content.append(f"\n- [{metadata['title']}]({url})")
+
+        return "\n".join(content)
+
+async def process_url(url, max_depth, max_pages):
+    """Process URL and generate llms.txt"""
+    try:
+        # Add https:// if not present
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        # Validate URL format
+        try:
+            result = urlparse(url)
+            if not all([result.scheme, result.netloc]):
+                return "", "Invalid URL format. Please enter a valid URL."
+        except:
+            return "", "Invalid URL format. Please enter a valid URL."
+
+        # Create crawler and process
+        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
+        await crawler.crawl_website(url)
+        content = crawler.generate_llms_txt()
+        
+        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
+    
+    except Exception as e:
+        return "", f"Error: {str(e)}"
+
+# Create custom theme
+theme = gr.themes.Soft(
+    primary_hue="blue",
+    font="Open Sans"
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=theme,     css="""
+    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
+    
+    .gradio-container {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+    
+    .gr-button {
+        font-family: 'Open Sans', sans-serif !important;
+        font-weight: 600 !important;
+    }
+
+    /* Primary color customization */
+    .primary-btn {
+        background-color: #2436d4 !important;
+        color: white !important;
+    }
+
+    .primary-btn:hover {
+        background-color: #1c2aa8 !important;
+    }
+    
+    [data-testid="textbox"] {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+""") as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Website URL", 
+            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
+            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            depth_input = gr.Slider(
+                minimum=1, 
+                maximum=5, 
+                value=3, 
+                step=1, 
+                label="Maximum Crawl Depth",
+                info="Higher values will result in more thorough but slower crawling"
+            )
+        with gr.Column():
+            pages_input = gr.Slider(
+                minimum=10, 
+                maximum=100, 
+                value=50, 
+                step=10, 
+                label="Maximum Pages to Crawl",
+                info="Higher values will result in more comprehensive but slower results"
+            )
+    
+    generate_btn = gr.Button("Generate llms.txt", variant="primary")
+    
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
+            lines=20,
+            max_lines=30,
+            show_copy_button=True,
+            container=True,
+            scale=2,
+            interactive=True
+        )
+    
+    status = gr.Textbox(label="Status")
+    
+    generate_btn.click(
+        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
+        inputs=[url_input, depth_input, pages_input],
+        outputs=[output, status]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    iface.launch()
+,         # Remove pipe and everything after
+            r'\s*-\s*.*
+
+    def is_valid_url(self, url, base_domain):
+        """Check if URL is valid and belongs to the same domain"""
+        try:
+            parsed = urlparse(url)
+            base_parsed = urlparse(base_domain)
+            return (parsed.netloc == base_parsed.netloc and
+                   parsed.scheme in ['http', 'https'] and
+                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
+        except:
+            return False
+
+    def extract_content(self, soup):
+        """Extract meaningful content from HTML"""
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
+            element.decompose()
+
+        # Get main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
+        if main_content:
+            return self.normalize_text(main_content.get_text(strip=True))
+        return self.normalize_text(soup.get_text(strip=True))
+
+    def get_page_metadata(self, soup, url):
+        """Extract metadata from the page"""
+        metadata = {
+            'title': None,
+            'description': None,
+            'importance': 0,
+            'category': 'Optional'
+        }
+
+        # Title extraction with cleaning
+        title = (
+            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
+            soup.find('title').text if soup.find('title') else
+            soup.find('h1').text if soup.find('h1') else
+            url.split('/')[-1]
+        )
+        metadata['title'] = self.clean_title(title)
+
+        # Description extraction with cleaning
+        description = (
+            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
+            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
+            ""
+        )
+        metadata['description'] = self.clean_description(description)
+
+        # Calculate importance and category
+        url_lower = url.lower()
+        if 'docs' in url_lower or 'documentation' in url_lower:
+            metadata['importance'] = 5
+            metadata['category'] = 'Docs'
+        elif 'api' in url_lower:
+            metadata['importance'] = 4
+            metadata['category'] = 'API'
+        elif 'guide' in url_lower or 'tutorial' in url_lower:
+            metadata['importance'] = 3
+            metadata['category'] = 'Guides'
+        elif 'example' in url_lower:
+            metadata['importance'] = 2
+            metadata['category'] = 'Examples'
+        elif 'blog' in url_lower:
+            metadata['importance'] = 1
+            metadata['category'] = 'Blog'
+
+        return metadata
+
+    async def crawl_page(self, url, depth, base_domain):
+        """Crawl a single page and extract information"""
+        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
+            return []
+
+        try:
+            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response.encoding = 'utf-8'
+            response.raise_for_status()
+            self.visited_urls.add(url)
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            content = self.extract_content(soup)
+            metadata = self.get_page_metadata(soup, url)
+            
+            self.url_content[url] = content
+            self.url_metadata[url] = metadata
+
+            # Find all links
+            links = []
+            for a in soup.find_all('a', href=True):
+                next_url = urljoin(url, a['href'])
+                if self.is_valid_url(next_url, base_domain):
+                    links.append(next_url)
+
+            return links
+
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            return []
+
+    async def crawl_website(self, start_url):
+        """Crawl website starting from the given URL"""
+        base_domain = start_url
+        queue = [(start_url, 0)]
+        seen = {start_url}
+
+        while queue and len(self.visited_urls) < self.max_pages:
+            current_url, depth = queue.pop(0)
+            
+            if depth > self.max_depth:
+                continue
+
+            links = await self.crawl_page(current_url, depth, base_domain)
+            
+            for link in links:
+                if link not in seen:
+                    seen.add(link)
+                    queue.append((link, depth + 1))
+
+    def generate_llms_txt(self):
+        """Generate llms.txt content from crawled data"""
+        # Sort URLs by importance
+        sorted_urls = sorted(
+            self.url_metadata.items(),
+            key=lambda x: (x[1]['importance'], x[0]),
+            reverse=True
+        )
+
+        if not sorted_urls:
+            return "No content was found to generate llms.txt"
+
+        # Group URLs by category
+        categorized_urls = defaultdict(list)
+        for url, metadata in sorted_urls:
+            categorized_urls[metadata['category']].append((url, metadata))
+
+        # Generate content
+        content = []
+        
+        # Add main title and description
+        main_metadata = sorted_urls[0][1]
+        content.append(f"# {main_metadata['title']}")
+        if main_metadata['description']:
+            content.append(f"\n> {main_metadata['description']}")
+        
+        # Add categorized sections
+        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
+        
+        for category in priority_order:
+            if category in categorized_urls:
+                content.append(f"\n## {category}")
+                for url, metadata in categorized_urls[category]:
+                    if metadata['description']:
+                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
+                    else:
+                        content.append(f"\n- [{metadata['title']}]({url})")
+
+        return "\n".join(content)
+
+async def process_url(url, max_depth, max_pages):
+    """Process URL and generate llms.txt"""
+    try:
+        # Add https:// if not present
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        # Validate URL format
+        try:
+            result = urlparse(url)
+            if not all([result.scheme, result.netloc]):
+                return "", "Invalid URL format. Please enter a valid URL."
+        except:
+            return "", "Invalid URL format. Please enter a valid URL."
+
+        # Create crawler and process
+        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
+        await crawler.crawl_website(url)
+        content = crawler.generate_llms_txt()
+        
+        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
+    
+    except Exception as e:
+        return "", f"Error: {str(e)}"
+
+# Create custom theme
+theme = gr.themes.Soft(
+    primary_hue="blue",
+    font="Open Sans"
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=theme,     css="""
+    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
+    
+    .gradio-container {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+    
+    .gr-button {
+        font-family: 'Open Sans', sans-serif !important;
+        font-weight: 600 !important;
+    }
+
+    /* Primary color customization */
+    .primary-btn {
+        background-color: #2436d4 !important;
+        color: white !important;
+    }
+
+    .primary-btn:hover {
+        background-color: #1c2aa8 !important;
+    }
+    
+    [data-testid="textbox"] {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+""") as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Website URL", 
+            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
+            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            depth_input = gr.Slider(
+                minimum=1, 
+                maximum=5, 
+                value=3, 
+                step=1, 
+                label="Maximum Crawl Depth",
+                info="Higher values will result in more thorough but slower crawling"
+            )
+        with gr.Column():
+            pages_input = gr.Slider(
+                minimum=10, 
+                maximum=100, 
+                value=50, 
+                step=10, 
+                label="Maximum Pages to Crawl",
+                info="Higher values will result in more comprehensive but slower results"
+            )
+    
+    generate_btn = gr.Button("Generate llms.txt", variant="primary")
+    
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
+            lines=20,
+            max_lines=30,
+            show_copy_button=True,
+            container=True,
+            scale=2,
+            interactive=True
+        )
+    
+    status = gr.Textbox(label="Status")
+    
+    generate_btn.click(
+        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
+        inputs=[url_input, depth_input, pages_input],
+        outputs=[output, status]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    iface.launch()
+,          # Remove dash and everything after
+            r'\s*:\s*.*
+
+    def is_valid_url(self, url, base_domain):
+        """Check if URL is valid and belongs to the same domain"""
+        try:
+            parsed = urlparse(url)
+            base_parsed = urlparse(base_domain)
+            return (parsed.netloc == base_parsed.netloc and
+                   parsed.scheme in ['http', 'https'] and
+                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
+        except:
+            return False
+
+    def extract_content(self, soup):
+        """Extract meaningful content from HTML"""
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
+            element.decompose()
+
+        # Get main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
+        if main_content:
+            return self.normalize_text(main_content.get_text(strip=True))
+        return self.normalize_text(soup.get_text(strip=True))
+
+    def get_page_metadata(self, soup, url):
+        """Extract metadata from the page"""
+        metadata = {
+            'title': None,
+            'description': None,
+            'importance': 0,
+            'category': 'Optional'
+        }
+
+        # Title extraction with cleaning
+        title = (
+            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
+            soup.find('title').text if soup.find('title') else
+            soup.find('h1').text if soup.find('h1') else
+            url.split('/')[-1]
+        )
+        metadata['title'] = self.clean_title(title)
+
+        # Description extraction with cleaning
+        description = (
+            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
+            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
+            ""
+        )
+        metadata['description'] = self.clean_description(description)
+
+        # Calculate importance and category
+        url_lower = url.lower()
+        if 'docs' in url_lower or 'documentation' in url_lower:
+            metadata['importance'] = 5
+            metadata['category'] = 'Docs'
+        elif 'api' in url_lower:
+            metadata['importance'] = 4
+            metadata['category'] = 'API'
+        elif 'guide' in url_lower or 'tutorial' in url_lower:
+            metadata['importance'] = 3
+            metadata['category'] = 'Guides'
+        elif 'example' in url_lower:
+            metadata['importance'] = 2
+            metadata['category'] = 'Examples'
+        elif 'blog' in url_lower:
+            metadata['importance'] = 1
+            metadata['category'] = 'Blog'
+
+        return metadata
+
+    async def crawl_page(self, url, depth, base_domain):
+        """Crawl a single page and extract information"""
+        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
+            return []
+
+        try:
+            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response.encoding = 'utf-8'
+            response.raise_for_status()
+            self.visited_urls.add(url)
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            content = self.extract_content(soup)
+            metadata = self.get_page_metadata(soup, url)
+            
+            self.url_content[url] = content
+            self.url_metadata[url] = metadata
+
+            # Find all links
+            links = []
+            for a in soup.find_all('a', href=True):
+                next_url = urljoin(url, a['href'])
+                if self.is_valid_url(next_url, base_domain):
+                    links.append(next_url)
+
+            return links
+
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            return []
+
+    async def crawl_website(self, start_url):
+        """Crawl website starting from the given URL"""
+        base_domain = start_url
+        queue = [(start_url, 0)]
+        seen = {start_url}
+
+        while queue and len(self.visited_urls) < self.max_pages:
+            current_url, depth = queue.pop(0)
+            
+            if depth > self.max_depth:
+                continue
+
+            links = await self.crawl_page(current_url, depth, base_domain)
+            
+            for link in links:
+                if link not in seen:
+                    seen.add(link)
+                    queue.append((link, depth + 1))
+
+    def generate_llms_txt(self):
+        """Generate llms.txt content from crawled data"""
+        # Sort URLs by importance
+        sorted_urls = sorted(
+            self.url_metadata.items(),
+            key=lambda x: (x[1]['importance'], x[0]),
+            reverse=True
+        )
+
+        if not sorted_urls:
+            return "No content was found to generate llms.txt"
+
+        # Group URLs by category
+        categorized_urls = defaultdict(list)
+        for url, metadata in sorted_urls:
+            categorized_urls[metadata['category']].append((url, metadata))
+
+        # Generate content
+        content = []
+        
+        # Add main title and description
+        main_metadata = sorted_urls[0][1]
+        content.append(f"# {main_metadata['title']}")
+        if main_metadata['description']:
+            content.append(f"\n> {main_metadata['description']}")
+        
+        # Add categorized sections
+        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
+        
+        for category in priority_order:
+            if category in categorized_urls:
+                content.append(f"\n## {category}")
+                for url, metadata in categorized_urls[category]:
+                    if metadata['description']:
+                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
+                    else:
+                        content.append(f"\n- [{metadata['title']}]({url})")
+
+        return "\n".join(content)
+
+async def process_url(url, max_depth, max_pages):
+    """Process URL and generate llms.txt"""
+    try:
+        # Add https:// if not present
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        # Validate URL format
+        try:
+            result = urlparse(url)
+            if not all([result.scheme, result.netloc]):
+                return "", "Invalid URL format. Please enter a valid URL."
+        except:
+            return "", "Invalid URL format. Please enter a valid URL."
+
+        # Create crawler and process
+        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
+        await crawler.crawl_website(url)
+        content = crawler.generate_llms_txt()
+        
+        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
+    
+    except Exception as e:
+        return "", f"Error: {str(e)}"
+
+# Create custom theme
+theme = gr.themes.Soft(
+    primary_hue="blue",
+    font="Open Sans"
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=theme,     css="""
+    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
+    
+    .gradio-container {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+    
+    .gr-button {
+        font-family: 'Open Sans', sans-serif !important;
+        font-weight: 600 !important;
+    }
+
+    /* Primary color customization */
+    .primary-btn {
+        background-color: #2436d4 !important;
+        color: white !important;
+    }
+
+    .primary-btn:hover {
+        background-color: #1c2aa8 !important;
+    }
+    
+    [data-testid="textbox"] {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+""") as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Website URL", 
+            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
+            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            depth_input = gr.Slider(
+                minimum=1, 
+                maximum=5, 
+                value=3, 
+                step=1, 
+                label="Maximum Crawl Depth",
+                info="Higher values will result in more thorough but slower crawling"
+            )
+        with gr.Column():
+            pages_input = gr.Slider(
+                minimum=10, 
+                maximum=100, 
+                value=50, 
+                step=10, 
+                label="Maximum Pages to Crawl",
+                info="Higher values will result in more comprehensive but slower results"
+            )
+    
+    generate_btn = gr.Button("Generate llms.txt", variant="primary")
+    
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
+            lines=20,
+            max_lines=30,
+            show_copy_button=True,
+            container=True,
+            scale=2,
+            interactive=True
+        )
+    
+    status = gr.Textbox(label="Status")
+    
+    generate_btn.click(
+        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
+        inputs=[url_input, depth_input, pages_input],
+        outputs=[output, status]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    iface.launch()
+,          # Remove colon and everything after
+            r'#.*
+
+    def is_valid_url(self, url, base_domain):
+        """Check if URL is valid and belongs to the same domain"""
+        try:
+            parsed = urlparse(url)
+            base_parsed = urlparse(base_domain)
+            return (parsed.netloc == base_parsed.netloc and
+                   parsed.scheme in ['http', 'https'] and
+                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
+        except:
+            return False
+
+    def extract_content(self, soup):
+        """Extract meaningful content from HTML"""
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
+            element.decompose()
+
+        # Get main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
+        if main_content:
+            return self.normalize_text(main_content.get_text(strip=True))
+        return self.normalize_text(soup.get_text(strip=True))
+
+    def get_page_metadata(self, soup, url):
+        """Extract metadata from the page"""
+        metadata = {
+            'title': None,
+            'description': None,
+            'importance': 0,
+            'category': 'Optional'
+        }
+
+        # Title extraction with cleaning
+        title = (
+            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
+            soup.find('title').text if soup.find('title') else
+            soup.find('h1').text if soup.find('h1') else
+            url.split('/')[-1]
+        )
+        metadata['title'] = self.clean_title(title)
+
+        # Description extraction with cleaning
+        description = (
+            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
+            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
+            ""
+        )
+        metadata['description'] = self.clean_description(description)
+
+        # Calculate importance and category
+        url_lower = url.lower()
+        if 'docs' in url_lower or 'documentation' in url_lower:
+            metadata['importance'] = 5
+            metadata['category'] = 'Docs'
+        elif 'api' in url_lower:
+            metadata['importance'] = 4
+            metadata['category'] = 'API'
+        elif 'guide' in url_lower or 'tutorial' in url_lower:
+            metadata['importance'] = 3
+            metadata['category'] = 'Guides'
+        elif 'example' in url_lower:
+            metadata['importance'] = 2
+            metadata['category'] = 'Examples'
+        elif 'blog' in url_lower:
+            metadata['importance'] = 1
+            metadata['category'] = 'Blog'
+
+        return metadata
+
+    async def crawl_page(self, url, depth, base_domain):
+        """Crawl a single page and extract information"""
+        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
+            return []
+
+        try:
+            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response.encoding = 'utf-8'
+            response.raise_for_status()
+            self.visited_urls.add(url)
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            content = self.extract_content(soup)
+            metadata = self.get_page_metadata(soup, url)
+            
+            self.url_content[url] = content
+            self.url_metadata[url] = metadata
+
+            # Find all links
+            links = []
+            for a in soup.find_all('a', href=True):
+                next_url = urljoin(url, a['href'])
+                if self.is_valid_url(next_url, base_domain):
+                    links.append(next_url)
+
+            return links
+
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            return []
+
+    async def crawl_website(self, start_url):
+        """Crawl website starting from the given URL"""
+        base_domain = start_url
+        queue = [(start_url, 0)]
+        seen = {start_url}
+
+        while queue and len(self.visited_urls) < self.max_pages:
+            current_url, depth = queue.pop(0)
+            
+            if depth > self.max_depth:
+                continue
+
+            links = await self.crawl_page(current_url, depth, base_domain)
+            
+            for link in links:
+                if link not in seen:
+                    seen.add(link)
+                    queue.append((link, depth + 1))
+
+    def generate_llms_txt(self):
+        """Generate llms.txt content from crawled data"""
+        # Sort URLs by importance
+        sorted_urls = sorted(
+            self.url_metadata.items(),
+            key=lambda x: (x[1]['importance'], x[0]),
+            reverse=True
+        )
+
+        if not sorted_urls:
+            return "No content was found to generate llms.txt"
+
+        # Group URLs by category
+        categorized_urls = defaultdict(list)
+        for url, metadata in sorted_urls:
+            categorized_urls[metadata['category']].append((url, metadata))
+
+        # Generate content
+        content = []
+        
+        # Add main title and description
+        main_metadata = sorted_urls[0][1]
+        content.append(f"# {main_metadata['title']}")
+        if main_metadata['description']:
+            content.append(f"\n> {main_metadata['description']}")
+        
+        # Add categorized sections
+        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
+        
+        for category in priority_order:
+            if category in categorized_urls:
+                content.append(f"\n## {category}")
+                for url, metadata in categorized_urls[category]:
+                    if metadata['description']:
+                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
+                    else:
+                        content.append(f"\n- [{metadata['title']}]({url})")
+
+        return "\n".join(content)
+
+async def process_url(url, max_depth, max_pages):
+    """Process URL and generate llms.txt"""
+    try:
+        # Add https:// if not present
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        # Validate URL format
+        try:
+            result = urlparse(url)
+            if not all([result.scheme, result.netloc]):
+                return "", "Invalid URL format. Please enter a valid URL."
+        except:
+            return "", "Invalid URL format. Please enter a valid URL."
+
+        # Create crawler and process
+        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
+        await crawler.crawl_website(url)
+        content = crawler.generate_llms_txt()
+        
+        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
+    
+    except Exception as e:
+        return "", f"Error: {str(e)}"
+
+# Create custom theme
+theme = gr.themes.Soft(
+    primary_hue="blue",
+    font="Open Sans"
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=theme,     css="""
+    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
+    
+    .gradio-container {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+    
+    .gr-button {
+        font-family: 'Open Sans', sans-serif !important;
+        font-weight: 600 !important;
+    }
+
+    /* Primary color customization */
+    .primary-btn {
+        background-color: #2436d4 !important;
+        color: white !important;
+    }
+
+    .primary-btn:hover {
+        background-color: #1c2aa8 !important;
+    }
+    
+    [data-testid="textbox"] {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+""") as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Website URL", 
+            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
+            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            depth_input = gr.Slider(
+                minimum=1, 
+                maximum=5, 
+                value=3, 
+                step=1, 
+                label="Maximum Crawl Depth",
+                info="Higher values will result in more thorough but slower crawling"
+            )
+        with gr.Column():
+            pages_input = gr.Slider(
+                minimum=10, 
+                maximum=100, 
+                value=50, 
+                step=10, 
+                label="Maximum Pages to Crawl",
+                info="Higher values will result in more comprehensive but slower results"
+            )
+    
+    generate_btn = gr.Button("Generate llms.txt", variant="primary")
+    
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
+            lines=20,
+            max_lines=30,
+            show_copy_button=True,
+            container=True,
+            scale=2,
+            interactive=True
+        )
+    
+    status = gr.Textbox(label="Status")
+    
+    generate_btn.click(
+        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
+        inputs=[url_input, depth_input, pages_input],
+        outputs=[output, status]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    iface.launch()
+,                # Remove hash and everything after
+            r'\s*\|.*
+
+    def is_valid_url(self, url, base_domain):
+        """Check if URL is valid and belongs to the same domain"""
+        try:
+            parsed = urlparse(url)
+            base_parsed = urlparse(base_domain)
+            return (parsed.netloc == base_parsed.netloc and
+                   parsed.scheme in ['http', 'https'] and
+                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
+        except:
+            return False
+
+    def extract_content(self, soup):
+        """Extract meaningful content from HTML"""
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
+            element.decompose()
+
+        # Get main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
+        if main_content:
+            return self.normalize_text(main_content.get_text(strip=True))
+        return self.normalize_text(soup.get_text(strip=True))
+
+    def get_page_metadata(self, soup, url):
+        """Extract metadata from the page"""
+        metadata = {
+            'title': None,
+            'description': None,
+            'importance': 0,
+            'category': 'Optional'
+        }
+
+        # Title extraction with cleaning
+        title = (
+            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
+            soup.find('title').text if soup.find('title') else
+            soup.find('h1').text if soup.find('h1') else
+            url.split('/')[-1]
+        )
+        metadata['title'] = self.clean_title(title)
+
+        # Description extraction with cleaning
+        description = (
+            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
+            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
+            ""
+        )
+        metadata['description'] = self.clean_description(description)
+
+        # Calculate importance and category
+        url_lower = url.lower()
+        if 'docs' in url_lower or 'documentation' in url_lower:
+            metadata['importance'] = 5
+            metadata['category'] = 'Docs'
+        elif 'api' in url_lower:
+            metadata['importance'] = 4
+            metadata['category'] = 'API'
+        elif 'guide' in url_lower or 'tutorial' in url_lower:
+            metadata['importance'] = 3
+            metadata['category'] = 'Guides'
+        elif 'example' in url_lower:
+            metadata['importance'] = 2
+            metadata['category'] = 'Examples'
+        elif 'blog' in url_lower:
+            metadata['importance'] = 1
+            metadata['category'] = 'Blog'
+
+        return metadata
+
+    async def crawl_page(self, url, depth, base_domain):
+        """Crawl a single page and extract information"""
+        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
+            return []
+
+        try:
+            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response.encoding = 'utf-8'
+            response.raise_for_status()
+            self.visited_urls.add(url)
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            content = self.extract_content(soup)
+            metadata = self.get_page_metadata(soup, url)
+            
+            self.url_content[url] = content
+            self.url_metadata[url] = metadata
+
+            # Find all links
+            links = []
+            for a in soup.find_all('a', href=True):
+                next_url = urljoin(url, a['href'])
+                if self.is_valid_url(next_url, base_domain):
+                    links.append(next_url)
+
+            return links
+
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            return []
+
+    async def crawl_website(self, start_url):
+        """Crawl website starting from the given URL"""
+        base_domain = start_url
+        queue = [(start_url, 0)]
+        seen = {start_url}
+
+        while queue and len(self.visited_urls) < self.max_pages:
+            current_url, depth = queue.pop(0)
+            
+            if depth > self.max_depth:
+                continue
+
+            links = await self.crawl_page(current_url, depth, base_domain)
+            
+            for link in links:
+                if link not in seen:
+                    seen.add(link)
+                    queue.append((link, depth + 1))
+
+    def generate_llms_txt(self):
+        """Generate llms.txt content from crawled data"""
+        # Sort URLs by importance
+        sorted_urls = sorted(
+            self.url_metadata.items(),
+            key=lambda x: (x[1]['importance'], x[0]),
+            reverse=True
+        )
+
+        if not sorted_urls:
+            return "No content was found to generate llms.txt"
+
+        # Group URLs by category
+        categorized_urls = defaultdict(list)
+        for url, metadata in sorted_urls:
+            categorized_urls[metadata['category']].append((url, metadata))
+
+        # Generate content
+        content = []
+        
+        # Add main title and description
+        main_metadata = sorted_urls[0][1]
+        content.append(f"# {main_metadata['title']}")
+        if main_metadata['description']:
+            content.append(f"\n> {main_metadata['description']}")
+        
+        # Add categorized sections
+        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
+        
+        for category in priority_order:
+            if category in categorized_urls:
+                content.append(f"\n## {category}")
+                for url, metadata in categorized_urls[category]:
+                    if metadata['description']:
+                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
+                    else:
+                        content.append(f"\n- [{metadata['title']}]({url})")
+
+        return "\n".join(content)
+
+async def process_url(url, max_depth, max_pages):
+    """Process URL and generate llms.txt"""
+    try:
+        # Add https:// if not present
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        # Validate URL format
+        try:
+            result = urlparse(url)
+            if not all([result.scheme, result.netloc]):
+                return "", "Invalid URL format. Please enter a valid URL."
+        except:
+            return "", "Invalid URL format. Please enter a valid URL."
+
+        # Create crawler and process
+        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
+        await crawler.crawl_website(url)
+        content = crawler.generate_llms_txt()
+        
+        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
+    
+    except Exception as e:
+        return "", f"Error: {str(e)}"
+
+# Create custom theme
+theme = gr.themes.Soft(
+    primary_hue="blue",
+    font="Open Sans"
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=theme,     css="""
+    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
+    
+    .gradio-container {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+    
+    .gr-button {
+        font-family: 'Open Sans', sans-serif !important;
+        font-weight: 600 !important;
+    }
+
+    /* Primary color customization */
+    .primary-btn {
+        background-color: #2436d4 !important;
+        color: white !important;
+    }
+
+    .primary-btn:hover {
+        background-color: #1c2aa8 !important;
+    }
+    
+    [data-testid="textbox"] {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+""") as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Website URL", 
+            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
+            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            depth_input = gr.Slider(
+                minimum=1, 
+                maximum=5, 
+                value=3, 
+                step=1, 
+                label="Maximum Crawl Depth",
+                info="Higher values will result in more thorough but slower crawling"
+            )
+        with gr.Column():
+            pages_input = gr.Slider(
+                minimum=10, 
+                maximum=100, 
+                value=50, 
+                step=10, 
+                label="Maximum Pages to Crawl",
+                info="Higher values will result in more comprehensive but slower results"
+            )
+    
+    generate_btn = gr.Button("Generate llms.txt", variant="primary")
+    
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
+            lines=20,
+            max_lines=30,
+            show_copy_button=True,
+            container=True,
+            scale=2,
+            interactive=True
+        )
+    
+    status = gr.Textbox(label="Status")
+    
+    generate_btn.click(
+        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
+        inputs=[url_input, depth_input, pages_input],
+        outputs=[output, status]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    iface.launch()
+,            # Remove pipe and everything after
+            r'\s*•.*
+
+    def is_valid_url(self, url, base_domain):
+        """Check if URL is valid and belongs to the same domain"""
+        try:
+            parsed = urlparse(url)
+            base_parsed = urlparse(base_domain)
+            return (parsed.netloc == base_parsed.netloc and
+                   parsed.scheme in ['http', 'https'] and
+                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
+        except:
+            return False
+
+    def extract_content(self, soup):
+        """Extract meaningful content from HTML"""
+        # Remove script and style elements
+        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
+            element.decompose()
+
+        # Get main content
+        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
+        if main_content:
+            return self.normalize_text(main_content.get_text(strip=True))
+        return self.normalize_text(soup.get_text(strip=True))
+
+    def get_page_metadata(self, soup, url):
+        """Extract metadata from the page"""
+        metadata = {
+            'title': None,
+            'description': None,
+            'importance': 0,
+            'category': 'Optional'
+        }
+
+        # Title extraction with cleaning
+        title = (
+            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
+            soup.find('title').text if soup.find('title') else
+            soup.find('h1').text if soup.find('h1') else
+            url.split('/')[-1]
+        )
+        metadata['title'] = self.clean_title(title)
+
+        # Description extraction with cleaning
+        description = (
+            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
+            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
+            ""
+        )
+        metadata['description'] = self.clean_description(description)
+
+        # Calculate importance and category
+        url_lower = url.lower()
+        if 'docs' in url_lower or 'documentation' in url_lower:
+            metadata['importance'] = 5
+            metadata['category'] = 'Docs'
+        elif 'api' in url_lower:
+            metadata['importance'] = 4
+            metadata['category'] = 'API'
+        elif 'guide' in url_lower or 'tutorial' in url_lower:
+            metadata['importance'] = 3
+            metadata['category'] = 'Guides'
+        elif 'example' in url_lower:
+            metadata['importance'] = 2
+            metadata['category'] = 'Examples'
+        elif 'blog' in url_lower:
+            metadata['importance'] = 1
+            metadata['category'] = 'Blog'
+
+        return metadata
+
+    async def crawl_page(self, url, depth, base_domain):
+        """Crawl a single page and extract information"""
+        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
+            return []
+
+        try:
+            response = requests.get(url, headers=self.headers, timeout=self.timeout)
+            response.encoding = 'utf-8'
+            response.raise_for_status()
+            self.visited_urls.add(url)
+
+            soup = BeautifulSoup(response.text, 'html.parser')
+            content = self.extract_content(soup)
+            metadata = self.get_page_metadata(soup, url)
+            
+            self.url_content[url] = content
+            self.url_metadata[url] = metadata
+
+            # Find all links
+            links = []
+            for a in soup.find_all('a', href=True):
+                next_url = urljoin(url, a['href'])
+                if self.is_valid_url(next_url, base_domain):
+                    links.append(next_url)
+
+            return links
+
+        except Exception as e:
+            logger.error(f"Error crawling {url}: {str(e)}")
+            return []
+
+    async def crawl_website(self, start_url):
+        """Crawl website starting from the given URL"""
+        base_domain = start_url
+        queue = [(start_url, 0)]
+        seen = {start_url}
+
+        while queue and len(self.visited_urls) < self.max_pages:
+            current_url, depth = queue.pop(0)
+            
+            if depth > self.max_depth:
+                continue
+
+            links = await self.crawl_page(current_url, depth, base_domain)
+            
+            for link in links:
+                if link not in seen:
+                    seen.add(link)
+                    queue.append((link, depth + 1))
+
+    def generate_llms_txt(self):
+        """Generate llms.txt content from crawled data"""
+        # Sort URLs by importance
+        sorted_urls = sorted(
+            self.url_metadata.items(),
+            key=lambda x: (x[1]['importance'], x[0]),
+            reverse=True
+        )
+
+        if not sorted_urls:
+            return "No content was found to generate llms.txt"
+
+        # Group URLs by category
+        categorized_urls = defaultdict(list)
+        for url, metadata in sorted_urls:
+            categorized_urls[metadata['category']].append((url, metadata))
+
+        # Generate content
+        content = []
+        
+        # Add main title and description
+        main_metadata = sorted_urls[0][1]
+        content.append(f"# {main_metadata['title']}")
+        if main_metadata['description']:
+            content.append(f"\n> {main_metadata['description']}")
+        
+        # Add categorized sections
+        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
+        
+        for category in priority_order:
+            if category in categorized_urls:
+                content.append(f"\n## {category}")
+                for url, metadata in categorized_urls[category]:
+                    if metadata['description']:
+                        content.append(f"\n- [{metadata['title']}]({url}): {metadata['description']}")
+                    else:
+                        content.append(f"\n- [{metadata['title']}]({url})")
+
+        return "\n".join(content)
+
+async def process_url(url, max_depth, max_pages):
+    """Process URL and generate llms.txt"""
+    try:
+        # Add https:// if not present
+        if not url.startswith(('http://', 'https://')):
+            url = 'https://' + url
+
+        # Validate URL format
+        try:
+            result = urlparse(url)
+            if not all([result.scheme, result.netloc]):
+                return "", "Invalid URL format. Please enter a valid URL."
+        except:
+            return "", "Invalid URL format. Please enter a valid URL."
+
+        # Create crawler and process
+        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
+        await crawler.crawl_website(url)
+        content = crawler.generate_llms_txt()
+        
+        return content, f"Successfully crawled {len(crawler.visited_urls)} pages."
+    
+    except Exception as e:
+        return "", f"Error: {str(e)}"
+
+# Create custom theme
+theme = gr.themes.Soft(
+    primary_hue="blue",
+    font="Open Sans"
+)
+
+# Create the Gradio interface
+with gr.Blocks(theme=theme,     css="""
+    @import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');
+    
+    .gradio-container {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+    
+    .gr-button {
+        font-family: 'Open Sans', sans-serif !important;
+        font-weight: 600 !important;
+    }
+
+    /* Primary color customization */
+    .primary-btn {
+        background-color: #2436d4 !important;
+        color: white !important;
+    }
+
+    .primary-btn:hover {
+        background-color: #1c2aa8 !important;
+    }
+    
+    [data-testid="textbox"] {
+        font-family: 'Open Sans', sans-serif !important;
+    }
+""") as iface:
+    gr.Markdown("# llms.txt Generator")
+    gr.Markdown("Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.")
+    
+    with gr.Row():
+        url_input = gr.Textbox(
+            label="Website URL", 
+            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
+            info="The URL will be automatically prefixed with https:// if no protocol is specified."
+        )
+    
+    with gr.Row():
+        with gr.Column():
+            depth_input = gr.Slider(
+                minimum=1, 
+                maximum=5, 
+                value=3, 
+                step=1, 
+                label="Maximum Crawl Depth",
+                info="Higher values will result in more thorough but slower crawling"
+            )
+        with gr.Column():
+            pages_input = gr.Slider(
+                minimum=10, 
+                maximum=100, 
+                value=50, 
+                step=10, 
+                label="Maximum Pages to Crawl",
+                info="Higher values will result in more comprehensive but slower results"
+            )
+    
+    generate_btn = gr.Button("Generate llms.txt", variant="primary")
+    
+    with gr.Row():
+        output = gr.Textbox(
+            label="Generated llms.txt Content",
+            lines=20,
+            max_lines=30,
+            show_copy_button=True,
+            container=True,
+            scale=2,
+            interactive=True
+        )
+    
+    status = gr.Textbox(label="Status")
+    
+    generate_btn.click(
+        fn=lambda url, depth, pages: asyncio.run(process_url(url, depth, pages)),
+        inputs=[url_input, depth_input, pages_input],
+        outputs=[output, status]
+    )
+
+# Launch the app
+if __name__ == "__main__":
+    iface.launch()
+,             # Remove bullet and everything after
+            r'^\s*Welcome to\s+',   # Remove "Welcome to" at start
+            r'docusaurus_skipToContent_fallback',  # Remove docusaurus fragments
+        ]
+        
+        for pattern in patterns:
+            title = re.sub(pattern, '', title)
+        
+        # Clean up whitespace
+        title = ' '.join(title.split())
+        return title.strip()
 
     def clean_description(self, desc):
         """Clean and format descriptions"""
         if not desc:
             return ""
+            
         desc = self.normalize_text(desc)
-        # Find the last complete sentence
+        
+        # Remove duplicate sentences
         sentences = re.split(r'(?<=[.!?])\s+', desc)
-        if sentences:
-            # Take up to two complete sentences
-            cleaned_desc = ' '.join(sentences[:2]).strip()
-            # Ensure it ends with proper punctuation
-            if not cleaned_desc[-1] in '.!?':
-                cleaned_desc += '.'
-            return cleaned_desc
-        return desc
+        unique_sentences = []
+        seen_sentences = set()
+        
+        for sentence in sentences:
+            sentence = sentence.strip()
+            sentence_lower = sentence.lower()
+            if sentence_lower not in seen_sentences and sentence:
+                if not sentence[-1] in '.!?':
+                    sentence += '.'
+                unique_sentences.append(sentence)
+                seen_sentences.add(sentence_lower)
+        
+        cleaned_desc = ' '.join(unique_sentences)
+        return cleaned_desc
 
     def is_valid_url(self, url, base_domain):
         """Check if URL is valid and belongs to the same domain"""