app.py

import gradio as gr
import requests
from bs4 import BeautifulSoup
import re
from urllib.parse import urljoin, urlparse
import markdown
from concurrent.futures import ThreadPoolExecutor
import asyncio
from collections import defaultdict
import time
import logging
import unicodedata

# Set up logging
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

class WebsiteCrawler:
    def __init__(self, max_depth=3, max_pages=50, timeout=30):
        self.max_depth = max_depth
        self.max_pages = max_pages
        self.timeout = timeout
        self.visited_urls = set()
        self.url_content = {}
        self.url_metadata = defaultdict(dict)
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        }

    def normalize_text(self, text):
        """Normalize text to handle encoding issues"""
        if not text:
            return ""
        # Normalize unicode characters
        text = unicodedata.normalize('NFKD', text)
        # Replace special quotes and dashes with standard characters
        text = text.replace('"', '"').replace('"', '"').replace(''', "'").replace('—', '-')
        # Remove any remaining non-ASCII characters
        text = text.encode('ascii', 'ignore').decode('ascii')
        # Clean up extra whitespace
        text = ' '.join(text.split())
        return text

    def is_valid_url(self, url, base_domain):
        """Check if URL is valid and belongs to the same domain"""
        try:
            parsed = urlparse(url)
            base_parsed = urlparse(base_domain)
            return (parsed.netloc == base_parsed.netloc and
                   parsed.scheme in ['http', 'https'] and
                   not url.endswith(('.pdf', '.jpg', '.png', '.gif', '.zip')))
        except:
            return False

    def extract_content(self, soup):
        """Extract meaningful content from HTML"""
        # Remove script and style elements
        for element in soup(['script', 'style', 'nav', 'footer', 'header']):
            element.decompose()

        # Get main content
        main_content = soup.find('main') or soup.find('article') or soup.find('div', {'class': re.compile(r'content|main', re.I)})
        if main_content:
            return self.normalize_text(main_content.get_text(strip=True))
        return self.normalize_text(soup.get_text(strip=True))

    def get_page_metadata(self, soup, url):
        """Extract metadata from the page"""
        metadata = {
            'title': None,
            'description': None,
            'importance': 0,
            'category': 'Optional'
        }

        # Title extraction with normalization
        title = (
            soup.find('meta', property='og:title')['content'] if soup.find('meta', property='og:title') else
            soup.find('title').text if soup.find('title') else
            soup.find('h1').text if soup.find('h1') else
            url.split('/')[-1]
        )
        metadata['title'] = self.normalize_text(title)

        # Description extraction with normalization
        description = (
            soup.find('meta', {'name': 'description'})['content'] if soup.find('meta', {'name': 'description'}) else
            soup.find('meta', property='og:description')['content'] if soup.find('meta', property='og:description') else
            ""
        )
        metadata['description'] = self.normalize_text(description)

        # Calculate importance based on various factors
        importance = 0
        if 'docs' in url.lower() or 'documentation' in url.lower():
            importance += 5
            metadata['category'] = 'Docs'
        if 'api' in url.lower():
            importance += 4
            metadata['category'] = 'API'
        if 'guide' in url.lower() or 'tutorial' in url.lower():
            importance += 3
            metadata['category'] = 'Guides'
        if 'example' in url.lower():
            importance += 2
            metadata['category'] = 'Examples'
        if 'blog' in url.lower():
            importance += 1
            metadata['category'] = 'Blog'

        metadata['importance'] = importance
        return metadata

    async def crawl_page(self, url, depth, base_domain):
        """Crawl a single page and extract information"""
        if depth > self.max_depth or url in self.visited_urls or len(self.visited_urls) >= self.max_pages:
            return []

        try:
            response = requests.get(url, headers=self.headers, timeout=self.timeout)
            response.encoding = 'utf-8'  # Explicitly set encoding
            response.raise_for_status()
            self.visited_urls.add(url)

            soup = BeautifulSoup(response.text, 'html.parser')
            content = self.extract_content(soup)
            metadata = self.get_page_metadata(soup, url)
            
            self.url_content[url] = content
            self.url_metadata[url] = metadata

            # Find all links
            links = []
            for a in soup.find_all('a', href=True):
                next_url = urljoin(url, a['href'])
                if self.is_valid_url(next_url, base_domain):
                    links.append(next_url)

            return links

        except Exception as e:
            logger.error(f"Error crawling {url}: {str(e)}")
            return []

    async def crawl_website(self, start_url):
        """Crawl website starting from the given URL"""
        base_domain = start_url
        queue = [(start_url, 0)]
        seen = {start_url}

        while queue and len(self.visited_urls) < self.max_pages:
            current_url, depth = queue.pop(0)
            
            if depth > self.max_depth:
                continue

            links = await self.crawl_page(current_url, depth, base_domain)
            
            for link in links:
                if link not in seen:
                    seen.add(link)
                    queue.append((link, depth + 1))

    def generate_llms_txt(self):
        """Generate llms.txt content from crawled data"""
        # Sort URLs by importance
        sorted_urls = sorted(
            self.url_metadata.items(),
            key=lambda x: (x[1]['importance'], x[0]),
            reverse=True
        )

        # Group URLs by category
        categorized_urls = defaultdict(list)
        for url, metadata in sorted_urls:
            categorized_urls[metadata['category']].append((url, metadata))

        # Generate content
        content = []
        
        # Add main title and description
        if sorted_urls:
            main_metadata = sorted_urls[0][1]
            content.append(f"# {main_metadata['title']}\n")
            content.append(f"> {main_metadata['description']}\n")
        
        # Add categorized sections
        priority_order = ['Docs', 'API', 'Guides', 'Examples', 'Blog', 'Optional']
        
        for category in priority_order:
            if category in categorized_urls:
                content.append(f"\n## {category}\n")
                for url, metadata in categorized_urls[category]:
                    title = metadata['title']
                    desc = metadata['description']
                    if desc:
                        content.append(f"- [{title}]({url}): {desc[:100]}...\n")
                    else:
                        content.append(f"- [{title}]({url})\n")

        return "\n".join(content)

async def process_url(url, max_depth, max_pages):
    """Process URL and generate llms.txt"""
    try:
        # Add https:// if not present
        if not url.startswith(('http://', 'https://')):
            url = 'https://' + url

        # Validate URL format
        try:
            result = urlparse(url)
            if not all([result.scheme, result.netloc]):
                return "", "Invalid URL format. Please enter a valid URL."
        except:
            return "", "Invalid URL format. Please enter a valid URL."

        # Create crawler and process
        crawler = WebsiteCrawler(max_depth=int(max_depth), max_pages=int(max_pages))
        await crawler.crawl_website(url)
        content = crawler.generate_llms_txt()
        
        return content, f"Successfully crawled {len(crawler.visited_urls)} pages. You can now copy the generated content."
    
    except Exception as e:
        return "", f"Error: {str(e)}"

# Create the Gradio interface with custom CSS for Open Sans font
css = """
@import url('https://fonts.googleapis.com/css2?family=Open+Sans:wght@400;600&display=swap');

body, .gradio-container {
    font-family: 'Open Sans', sans-serif !important;
}

.gr-box {
    border-radius: 8px !important;
    border: 1px solid #e5e7eb !important;
}

.gr-button {
    font-family: 'Open Sans', sans-serif !important;
    font-weight: 600 !important;
}

.gr-input {
    font-family: 'Open Sans', sans-serif !important;
}
"""

# Create the Gradio interface
iface = gr.Interface(
    fn=lambda url, max_depth, max_pages: asyncio.run(process_url(url, max_depth, max_pages)),
    inputs=[
        gr.Textbox(
            label="Website URL", 
            placeholder="Enter the website URL (e.g., example.com or https://example.com)",
            info="The URL will be automatically prefixed with https:// if no protocol is specified."
        ),
        gr.Slider(
            minimum=1, 
            maximum=5, 
            value=3, 
            step=1, 
            label="Maximum Crawl Depth",
            info="Higher values will result in more thorough but slower crawling"
        ),
        gr.Slider(
            minimum=10, 
            maximum=100, 
            value=50, 
            step=10, 
            label="Maximum Pages to Crawl",
            info="Higher values will result in more comprehensive but slower results"
        )
    ],
    outputs=[
        gr.Textbox(
            label="Generated llms.txt Content", 
            lines=20,
            info="Copy this content to create your llms.txt file"
        ),
        gr.Textbox(label="Status")
    ],
    title="llms.txt Generator",
    description="Generate an llms.txt file from a website following the specification. The tool crawls the website and creates a structured markdown file suitable for LLMs.",
    theme=gr.themes.Soft(),
    css=css
)

# Launch the app
if __name__ == "__main__":
    iface.launch()