Spaces:
Sleeping
Sleeping
File size: 2,176 Bytes
a23f3fb |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
import requests
from bs4 import BeautifulSoup
import pandas as pd
df = pd.DataFrame(columns=['page_url', 'image_url', 'author', 'title', 'annotation'])
def extract_data_from_page(page_number):
url = f'https://www.chitai-gorod.ru/catalog/books/hudozhestvennaya-literatura-110001?page={page_number}'
response = requests.get(url)
soup = BeautifulSoup(response.content, 'html.parser')
books = soup.find_all('article', class_='product-card')
data = []
for book in books:
try:
book_url = book.find('a', class_='product-card__picture')['href']
title = book.find('div', class_='product-title__head').get_text(strip=True)
author = book.find('div', class_='product-title__author').get_text(strip=True)
absolute_url = f'https://www.chitai-gorod.ru{book_url}'
data.append({'page_url': absolute_url, 'title': title, 'author': author})
except Exception as e:
print(f"Error processing book: {e}")
return data
for page in range(2, 201):
print(f"Processing page {page}...")
page_data = extract_data_from_page(page)
df = pd.concat([df, pd.DataFrame(page_data)], ignore_index=True)
if len(df) >= 5000:
break
df = df.head(5000)
def extract_book_details(book_url):
try:
response = requests.get(book_url)
soup = BeautifulSoup(response.content, 'html.parser')
image_tag = soup.find('meta', {'name': 'og:image'})
image_url = image_tag['content'] if image_tag else None
annotation_tag = soup.find('div', {'itemprop': 'description'})
annotation = annotation_tag.get_text(strip=True) if annotation_tag else None
return image_url, annotation
except Exception as e:
print(f"Error extracting details from {book_url}: {e}")
return None, None
for idx, row in df.head(5000).iterrows():
print(f"Fetching details for {row['page_url']}...")
image_url, annotation = extract_book_details(row['page_url'])
df.at[idx, 'image_url'] = image_url
df.at[idx, 'annotation'] = annotation
df.to_csv('books_data_with_details.csv', index=False) |