Update tools/visit_webpage.py

tools/visit_webpage.py

@@ -3,6 +3,7 @@ from smolagents.tools import Tool
 import requests
 import markdownify
 import smolagents
+import re
 
 class VisitWebpageTool(Tool):
     name = "visit_webpage"
@@ -22,8 +23,21 @@ class VisitWebpageTool(Tool):
                 "You must install packages `markdownify` and `requests` to run this tool: for instance run `pip install markdownify requests`."
             ) from e
         try:
+            headers = {
+                "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.1 Safari/605.1.15",
+                "referer": "https://duckduckgo.com/",
+                "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                'accept-language': 'en-US,en;q=0.9',
+                'accept-Encoding': 'gzip, deflate, br',
+                'priority': 'u=0, i',
+                'referer': 'https://duckduckgo.com/',
+                'sec-fetch-dest': 'document',
+                'sec-fetch-mode': 'navigate',
+                'sec-fetch-site': 'cross-site',
+            }
+            
             # Send a GET request to the URL with a 20-second timeout
-            response = requests.get(url, timeout=20)
+            response = requests.get(url, timeout=20, allow_redirects=True, headers=headers)
             response.raise_for_status()  # Raise an exception for bad status codes
 
             # Convert the HTML content to Markdown
@@ -32,7 +46,7 @@ class VisitWebpageTool(Tool):
             # Remove multiple line breaks
             markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
 
-            return truncate_content(markdown_content, 10000)
+            return truncate_content(markdown_content, 50000)
 
         except requests.exceptions.Timeout:
             return "The request timed out. Please try again later or check the URL."
@@ -42,4 +56,4 @@ class VisitWebpageTool(Tool):
             return f"An unexpected error occurred: {str(e)}"
 
     def __init__(self, *args, **kwargs):
-        self.is_initialized = False
+        self.is_initialized = False