Update tools/visit_webpage.py

tools/visit_webpage.py

@@ -1,8 +1,5 @@
-from typing import Any, Optional
 from smolagents.tools import Tool
-import requests
-import markdownify
-import smolagents
+from selenium import webdriver
 import re
 
 class VisitWebpageTool(Tool):
@@ -23,25 +20,14 @@ class VisitWebpageTool(Tool):
                 "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
             ) from e
         try:
-            headers = {
-                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
-                "referer": "https://duckduckgo.com/",
-                "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-                'accept-language': 'en-US,en;q=0.9',
-                'accept-Encoding': 'gzip, deflate, br',
-                'priority': 'u=0, i',
-                'referer': 'https://duckduckgo.com/',
-                'sec-fetch-dest': 'document',
-                'sec-fetch-mode': 'navigate',
-                'sec-fetch-site': 'cross-site',
-            }
-            
-            # Send a GET request to the URL with a 20-second timeout
-            response = requests.get(url, timeout=20, allow_redirects=True, headers=headers)
-            response.raise_for_status()  # Raise an exception for bad status codes
+            options = webdriver.ChromeOptions()
+            options.add_argument("--headless")  # Run in headless mode
+
+            driver = webdriver.Chrome(options=options)
+            driver.get(url)
 
             # Convert the HTML content to Markdown
-            markdown_content = markdownify(response.text).strip()
+            markdown_content = markdownify(driver.page_source).strip()
 
             # Remove multiple line breaks
             markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)