Update app.py

app.py

@@ -10,6 +10,8 @@ import filetype
 import requests
 import os
 import mimetypes
+from bs4 import BeautifulSoup
+from urllib.parse import urljoin
 
 # Constants
 CHUNK_SIZE = 32000
@@ -32,7 +34,6 @@ def clean_text(content):
     content = re.sub(r'\s+', ' ', content)
     return content
 
-
 def split_content(content, chunk_size=CHUNK_SIZE):
     """Splits content into chunks of a specified size."""
     chunks = []
@@ -92,7 +93,6 @@ def extract_text_from_pptx(pptx_data, clean=True):
     return text, len(text)
 
 def read_document(file_path, clean=True):
-
     with open(file_path, "rb") as f:
         file_content = f.read()
 
@@ -154,7 +154,17 @@ def read_document(file_path, clean=True):
             return extract_text_from_pptx(file_content, clean)
         except Exception as e:
             return f"Error reading PPTX: {e}", 0
-
+    elif mime == "text/html":  # Handle HTML content
+        try:
+            soup = BeautifulSoup(file_content, 'html.parser')
+            structured_data = {
+                "Texts": extract_texts(soup),
+                "Links": extract_links(soup, ""), 
+                "Images": extract_images(soup, "")
+            }
+            return format_detailed_output(structured_data), 0
+        except Exception as e:
+            return f"Error parsing HTML content: {e}", 0
     else:
         try:
             content = file_content.decode('utf-8')
@@ -165,43 +175,111 @@ def read_document(file_path, clean=True):
             return f"Error reading file: {e}", 0
 
 def download_and_process_file(url, clean=True):
-  """Downloads a file from a URL and returns the local file path."""
-  if not url.startswith("http://") and not url.startswith("https://"):
-    url = "http://" + url  # Prepend "http://" if not present
-
-  try:
-    response = requests.get(url, stream=True)
-    response.raise_for_status()  # Raise an exception for bad status codes
-
-    # Generate a safe and unique temporary filename
-    original_filename = os.path.basename(url)
-    # Remove invalid characters from filename
-    safe_filename = re.sub(r'[^\w\-_\. ]', '_', original_filename) 
-    temp_filename = f"{safe_filename}"
-
-    # Infer file extension from content type
-    content_type = response.headers['content-type']
-    ext = mimetypes.guess_extension(content_type)
-    if ext and not temp_filename.endswith(ext):  # Append extension if not already present
-        temp_filename += ext
-
-    with open(temp_filename, 'wb') as f:
-      for chunk in response.iter_content(chunk_size=8192000):
-        f.write(chunk)
-
-    # Check if it's an image type
-    kind = filetype.guess(temp_filename)
-    if kind and kind.mime.startswith('image/'):
-      return f"![]({url})", 0 # Return markdown image syntax if it's an image
+    """Downloads a file from a URL and returns the local file path."""
+    if not url.startswith("http://") and not url.startswith("https://"):
+        url = "http://" + url  # Prepend "http://" if not present
+
+    try:
+        response = requests.get(url, stream=True, timeout=10)
+        response.raise_for_status()  # Raise an exception for bad status codes
+
+        # Generate a safe and unique temporary filename
+        original_filename = os.path.basename(url)
+        # Remove invalid characters from filename
+        safe_filename = re.sub(r'[^\w\-_\. ]', '_', original_filename)
+        temp_filename = f"{safe_filename}"
+
+        # Infer file extension from content type
+        content_type = response.headers['content-type']
+        ext = mimetypes.guess_extension(content_type)
+        if ext and not temp_filename.endswith(ext):  # Append extension if not already present
+            temp_filename += ext
+
+        with open(temp_filename, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192000):
+                f.write(chunk)
+
+        # Check if it's an image type
+        kind = filetype.guess(temp_filename)
+        if kind and kind.mime.startswith('image/'):
+            return f"![]({url})", 0  # Return markdown image syntax if it's an image
+        else:
+            return read_document(temp_filename, clean)  # Otherwise, process as a document
+
+    except requests.exceptions.MissingSchema:
+        return "Error: Invalid URL format. Even after adding 'http://', the URL is still invalid.", 0
+    except requests.exceptions.ConnectionError:
+        return "Error: Could not connect to the server. Please check your internet connection.", 0
+    except requests.exceptions.Timeout:
+        return "Error: Connection timed out while trying to fetch the URL.", 0
+    except requests.exceptions.RequestException as e:
+        return f"Error downloading file: {e}", 0
+
+# --- Web Page Content Extraction Functions (from previous code) ---
+
+def extract_texts(soup):
+    """Extracts all text content from the soup."""
+    return [text for text in soup.stripped_strings]
+
+def extract_links(soup, base_url):
+    """Extracts all valid links from the soup."""
+    links = []
+    for link in soup.find_all('a', href=True):
+        href = link['href']
+        # Use urljoin to create an absolute URL
+        full_url = urljoin(base_url, href) if not href.startswith(("http://", "https://")) else href
+        link_text = link.get_text(strip=True) or "No Text"
+        links.append({"Text": link_text, "URL": full_url})
+    return links
+
+def extract_images(soup, base_url):
+    """Extracts all valid image URLs and their alt text from the soup."""
+    images = []
+    for img in soup.find_all('img', src=True):
+        img_url = img['src']
+        # Use urljoin to create an absolute URL
+        full_img_url = urljoin(base_url, img_url) if not img_url.startswith(("http://", "https://")) else img_url
+        alt_text = img.get('alt', 'No Alt Text')
+        images.append({"Alt Text": alt_text, "Image URL": full_img_url})
+    return images
+
+def fetch_page_content(url):
+    """Fetches the content of the page at the given URL."""
+    try:
+        response = requests.get(url, timeout=10)
+        response.raise_for_status()
+        return response.text
+    except requests.exceptions.RequestException as e:
+        return f"Error fetching the URL: {e}"
+
+def format_detailed_output(structured_data):
+    """Formats the structured data into a Markdown string."""
+    result = "### Structured Page Content\n\n"
+    result += "**Texts:**\n" + (" ".join(structured_data["Texts"]) if structured_data["Texts"] else "No textual content found.") + "\n\n"
+    result += "**Links:**\n"
+    if structured_data["Links"]:
+        result += "\n".join(f"[{link['Text']}]({link['URL']})" for link in structured_data["Links"]) + "\n"
     else:
-      return read_document(temp_filename, clean) # Otherwise, process as a document
-
-  except requests.exceptions.MissingSchema:
-    return "Error: Invalid URL format. Even after adding 'http://', the URL is still invalid.", 0
-  except requests.exceptions.ConnectionError:
-    return "Error: Could not connect to the server. Please check your internet connection.", 0
-  except requests.exceptions.RequestException as e:
-    return f"Error downloading file: {e}", 0
+        result += "No links found.\n"
+    result += "**Images:**\n"
+    if structured_data["Images"]:
+        result += "\n".join(f"![{img['Alt Text']}]({img['Image URL']})" for img in structured_data["Images"]) + "\n"
+    else:
+        result += "No images found.\n"
+    return result
+
+def extract_page_content(url):
+    """Extracts and formats the content of the page at the given URL."""
+    page_content = fetch_page_content(url)
+    if "Error" in page_content:
+        return page_content
+    soup = BeautifulSoup(page_content, 'html.parser')
+    structured_data = {
+        "Texts": extract_texts(soup),
+        "Links": extract_links(soup, url),  # Pass the base URL
+        "Images": extract_images(soup, url)  # Pass the base URL
+    }
+    return format_detailed_output(structured_data)
 
 # --- Gradio Interface ---
 
@@ -212,13 +290,11 @@ iface = gr.Interface(
         gr.Checkbox(label="Clean Text", value=True),
     ],
     outputs=[
-        gr.Textbox(label="Document Content/Image Markdown"),
+        gr.Textbox(label="Document Content/Image Markdown/Web Page Content"),
         gr.Number(label="Document Length (characters)"),
     ],
-    title="Enhanced File Processor for Hugging Face Chat Tools",
-    description="Enter the URL of site and extract its content"
-                "This tool is designed for use with Hugging Face Chat Tools: "
-                "[https://hf.co/chat/tools/66f1a8159d41ad4398ebb711](https://hf.co/chat/tools/66f1a8159d41ad4398ebb711)",
+    title="Enhanced File and Web Page Processor for Hugging Face Chat Tools",
+    description="Enter the URL of an image, video, document, or web page. The tool will handle it accordingly: images will be displayed as Markdown, documents will have their text extracted, and web pages will have their content structured and displayed. This tool is designed for use with Hugging Face Chat Tools. \n [https://hf.co/chat/tools/66f1a8159d41ad4398ebb711](https://hf.co/chat/tools/66f1a8159d41ad4398ebb711)",
     concurrency_limit=None
 )