Spaces:
Sleeping
Sleeping
File size: 1,364 Bytes
b7a1a13 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 |
import requests
from bs4 import BeautifulSoup
def get_content(url):
# Make a request to Prerender.io
response = requests.get(url, headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36'})
# Process the response
html_content = response.text
# Parse the HTML content
soup = BeautifulSoup(html_content, 'html.parser')
# Extract the title
title = soup.title.string if soup.title else ''
# Extract meta description
meta_description = soup.find('meta', attrs={'name': 'description'})
description = meta_description['content'] if meta_description else ''
# Extract headings
headings = [h.get_text() for h in soup.find_all(['h1', 'h2', 'h3'])]
# Extract main paragraphs
paragraphs = [p.get_text() for p in soup.find_all('p')]
headings = ' '.join(headings)
paragraphs = ' '.join(paragraphs)
headings = headings.replace("\n", "").replace("\t", "").replace(",", ";")
paragraphs = headings.replace("\n", "").replace("\t", "").replace(",", ";")
description = description.replace(",", ";")
title = title.replace(",", ";")
text = "[title] "+ title + "\n [description]" + description
# return {"url": url, "title":title, "description": description, "paragraphs": paragraphs, "headings":headings, "text": text , "summary": ""}
return text, title |